JUnit version 4.13.2
.00:28:04.654 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783470484646
00:28:04.707 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
00:28:04.737 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:28:04.747 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbWsn41/test-tmp
00:28:04.770 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
00:28:04.772 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbWsn41/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
00:28:04.974 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

00:28:04.988 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:28:04.989 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

00:28:04.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.990804  4022 mini_oidc.cc:150] Starting JWKS server
00:28:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.993129  4022 webserver.cc:534] Webserver started at http://127.0.0.1:37913/ using document root <none> and password file <none>
00:28:04.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.993321  4022 mini_oidc.cc:186] Starting OIDC Discovery server
00:28:04.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.993621  4022 webserver.cc:534] Webserver started at http://127.0.0.1:34931/ using document root <none> and password file <none>
00:28:05.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:28:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
00:28:05.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:28:05Z Disabled control of system clock
00:28:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.011581  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/wal
00:28:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data
00:28:05.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:05.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:05.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/logs
00:28:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data/info.pb
00:28:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:05.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:05.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:05.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:05.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:05.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:05.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:05.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.190:45669
00:28:05.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.190
00:28:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37913/jwks.json with env {}
00:28:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.109517  4033 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.109683  4033 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.109722  4033 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:05.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.113171  4033 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:28:05.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.113220  4033 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:05.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.113240  4033 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:05.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.113255  4033 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:28:05.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.113272  4033 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:05.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.113286  4033 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37913/jwks.json
00:28:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.117306  4033 master_runner.cc:387] Master server non-default flags:
00:28:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data
00:28:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/wal
00:28:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.190:45669
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37913/jwks.json
00:28:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data/info.pb
00:28:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.190
00:28:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4033
00:28:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/logs
00:28:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.118505  4033 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.119719  4033 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.126338  4033 server_base.cc:1061] running on GCE node
00:28:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.126415  4041 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.126448  4038 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.126618  4039 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.127724  4033 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:05.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.128778  4033 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:05.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.129956  4033 hybrid_clock.cc:648] HybridClock initialized: now 1783470485129934 us; error 35 us; skew 500 ppm
00:28:05.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.131729  4033 webserver.cc:534] Webserver started at http://127.3.237.190:33407/ using document root <none> and password file <none>
00:28:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.132411  4033 fs_manager.cc:362] Metadata directory not provided
00:28:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.132495  4033 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:05.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.132714  4033 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:28:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.134351  4033 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data/instance:
00:28:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48fcb917bd34a83ad007372c20f61e1"
00:28:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.134886  4033 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/wal/instance:
00:28:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48fcb917bd34a83ad007372c20f61e1"
00:28:05.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.138528  4033 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.006s	sys 0.000s
00:28:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.140777  4047 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.141697  4033 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
00:28:05.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.141829  4033 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/wal
00:28:05.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48fcb917bd34a83ad007372c20f61e1"
00:28:05.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.141913  4033 fs_report.cc:389] FS layout report
00:28:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/wal
00:28:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/wal
00:28:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data/data
00:28:05.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:05.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:05.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.164165  4033 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:05.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.164870  4033 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:05.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.165136  4033 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:05.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.173803  4099 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.190:45669 every 8 connection(s)
00:28:05.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.173807  4033 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.190:45669
00:28:05.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.175032  4033 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/data/info.pb
00:28:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.178423  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4033
00:28:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.178575  4022 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-0/wal/instance
00:28:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.178779  4100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:05.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.180354  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/wal
00:28:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data
00:28:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/logs
00:28:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data/info.pb
00:28:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:05.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.189:40855
00:28:05.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.189
00:28:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:05.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:05.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37913/jwks.json with env {}
00:28:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.182772  4100 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.199059  4100 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.199956  4051 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.189:40855: connect: Connection refused (error 111)
00:28:05.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.201159  4100 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.189:40855: Network error: Client connection negotiation failed: client connection to 127.3.237.189:40855: connect: Connection refused (error 111)
00:28:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.250357  4100 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } attempt: 1
00:28:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.251848  4100 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.189:40855: Network error: Client connection negotiation failed: client connection to 127.3.237.189:40855: connect: Connection refused (error 111)
00:28:05.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.288247  4102 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:05.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.288416  4102 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:05.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.288456  4102 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:05.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.291702  4102 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:28:05.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.291752  4102 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:05.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.291771  4102 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:05.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.291788  4102 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:28:05.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.291805  4102 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:05.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.291875  4102 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37913/jwks.json
00:28:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.296639  4102 master_runner.cc:387] Master server non-default flags:
00:28:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data
00:28:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/wal
00:28:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.189:40855
00:28:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37913/jwks.json
00:28:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data/info.pb
00:28:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.189
00:28:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4102
00:28:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/logs
00:28:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.297598  4102 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.298946  4102 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.305078  4113 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.305081  4110 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.305081  4111 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.305583  4102 server_base.cc:1061] running on GCE node
00:28:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.306115  4102 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.307067  4102 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.308216  4102 hybrid_clock.cc:648] HybridClock initialized: now 1783470485308164 us; error 71 us; skew 500 ppm
00:28:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.309846  4102 webserver.cc:534] Webserver started at http://127.3.237.189:46297/ using document root <none> and password file <none>
00:28:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.310436  4102 fs_manager.cc:362] Metadata directory not provided
00:28:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.310518  4102 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.310732  4102 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:28:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.312348  4102 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data/instance:
00:28:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f"
00:28:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.312857  4102 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/wal/instance:
00:28:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f"
00:28:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.316159  4102 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.000s
00:28:05.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.318249  4119 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:05.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.319346  4102 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:05.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.319475  4102 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/wal
00:28:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f"
00:28:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.319553  4102 fs_report.cc:389] FS layout report
00:28:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/wal
00:28:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/wal
00:28:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data/data
00:28:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:05.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.322834  4100 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } attempt: 2
00:28:05.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.324211  4100 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.189:40855: Network error: Client connection negotiation failed: client connection to 127.3.237.189:40855: connect: Connection refused (error 111)
00:28:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.332674  4102 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.333163  4102 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.333295  4102 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.339709  4102 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.189:40855
00:28:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.339720  4171 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.189:40855 every 8 connection(s)
00:28:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.340744  4102 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/data/info.pb
00:28:05.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.343209  4172 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:05.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.345994  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4102
00:28:05.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.346107  4022 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-1/wal/instance
00:28:05.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.345854  4172 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.346827  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/wal
00:28:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data
00:28:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/logs
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data/info.pb
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.188:36829
00:28:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.188
00:28:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37913/jwks.json with env {}
00:28:05.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.355738  4172 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.364630  4172 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.365389  4120 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.188:36829: connect: Connection refused (error 111)
00:28:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.366813  4172 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.188:36829: Network error: Client connection negotiation failed: client connection to 127.3.237.188:36829: connect: Connection refused (error 111)
00:28:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.415407  4100 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } attempt: 3
00:28:05.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.416076  4172 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } attempt: 1
00:28:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.417553  4172 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.188:36829: Network error: Client connection negotiation failed: client connection to 127.3.237.188:36829: connect: Connection refused (error 111)
00:28:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.420612  4100 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.422103  4100 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.188:36829: Network error: Client connection negotiation failed: client connection to 127.3.237.188:36829: connect: Connection refused (error 111)
00:28:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.451349  4174 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:05.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.451545  4174 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:05.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.451591  4174 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:05.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.453280  4100 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } attempt: 1
00:28:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.454772  4100 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.188:36829: Network error: Client connection negotiation failed: client connection to 127.3.237.188:36829: connect: Connection refused (error 111)
00:28:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.455320  4174 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:28:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.455365  4174 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.455385  4174 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.455402  4174 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:28:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.455423  4174 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.455439  4174 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37913/jwks.json
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.460165  4174 master_runner.cc:387] Master server non-default flags:
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/wal
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.188:36829
00:28:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37913/jwks.json
00:28:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data/info.pb
00:28:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.188
00:28:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4174
00:28:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/logs
00:28:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.461239  4174 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.462517  4174 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.468601  4182 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.469121  4183 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.468616  4185 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.472527  4174 server_base.cc:1061] running on GCE node
00:28:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.472980  4174 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.473809  4174 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.474987  4174 hybrid_clock.cc:648] HybridClock initialized: now 1783470485474967 us; error 37 us; skew 500 ppm
00:28:05.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.480302  4174 webserver.cc:534] Webserver started at http://127.3.237.188:46363/ using document root <none> and password file <none>
00:28:05.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.481005  4174 fs_manager.cc:362] Metadata directory not provided
00:28:05.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.481103  4174 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:05.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.481345  4174 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:28:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.483040  4174 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data/instance:
00:28:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49d83bbc22074419af110d15665c1974"
00:28:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.483580  4174 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/wal/instance:
00:28:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49d83bbc22074419af110d15665c1974"
00:28:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.485702  4172 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } attempt: 2
00:28:05.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.487046  4172 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.237.188:36829: Network error: Client connection negotiation failed: client connection to 127.3.237.188:36829: connect: Connection refused (error 111)
00:28:05.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.487094  4174 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.002s	sys 0.000s
00:28:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.489141  4191 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.490163  4174 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.490257  4174 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/wal
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49d83bbc22074419af110d15665c1974"
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.490319  4174 fs_report.cc:389] FS layout report
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/wal
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/wal
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data/data
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.502107  4174 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.502729  4174 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.502866  4174 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.509867  4174 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.188:36829
00:28:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.509884  4243 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.188:36829 every 8 connection(s)
00:28:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.510746  4174 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/data/info.pb
00:28:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.512782  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4174
00:28:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.512918  4022 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/master-2/wal/instance
00:28:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.513819  4244 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.517313  4244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.525992  4244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.529945  4100 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } attempt: 2
00:28:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.530148  4244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } has no permanent_uuid. Determining permanent_uuid...
00:28:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.537223  4100 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1: Bootstrap starting.
00:28:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.537456  4244 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: Bootstrap starting.
00:28:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.539253  4100 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1: Neither blocks nor log segments found. Creating new log.
00:28:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.539790  4244 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: Neither blocks nor log segments found. Creating new log.
00:28:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.540218  4100 log.cc:826] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1: Log is configured to *not* fsync() on all Append() calls
00:28:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.540882  4244 log.cc:826] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: Log is configured to *not* fsync() on all Append() calls
00:28:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.541744  4100 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1: No bootstrap required, opened a new log
00:28:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.542321  4244 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: No bootstrap required, opened a new log
00:28:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.544584  4100 raft_consensus.cc:359] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.544709  4100 raft_consensus.cc:385] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.544747  4100 raft_consensus.cc:740] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c48fcb917bd34a83ad007372c20f61e1, State: Initialized, Role: FOLLOWER
00:28:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.545118  4100 consensus_queue.cc:260] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.545069  4244 raft_consensus.cc:359] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.545255  4244 raft_consensus.cc:385] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.545323  4244 raft_consensus.cc:740] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 49d83bbc22074419af110d15665c1974, State: Initialized, Role: FOLLOWER
00:28:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.545745  4244 consensus_queue.cc:260] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.546165  4100 sys_catalog.cc:565] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: configured and running, proceeding with master startup.
00:28:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.545955  4255 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.546618  4255 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: This master's current role is: FOLLOWER
00:28:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.546752  4244 sys_catalog.cc:565] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [sys.catalog]: configured and running, proceeding with master startup.
00:28:05.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.552343  4278 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:05.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.552654  4278 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.553172  4256 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.553489  4256 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [sys.catalog]: This master's current role is: FOLLOWER
00:28:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.556054  4277 catalog_manager.cc:1593] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.556120  4277 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.554234  4033 master_runner.cc:187] Error getting master registration for 127.3.237.189:40855: OK, instance_id { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" instance_seqno: 1783470485338074 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:28:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.556619  4033 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:28:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.557324  4174 master_runner.cc:187] Error getting master registration for 127.3.237.189:40855: OK, instance_id { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" instance_seqno: 1783470485338074 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:28:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.559470  4174 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:28:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.578235  4172 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } attempt: 3
00:28:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.585111  4172 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: Bootstrap starting.
00:28:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.587689  4172 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: Neither blocks nor log segments found. Creating new log.
00:28:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.588500  4172 log.cc:826] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: Log is configured to *not* fsync() on all Append() calls
00:28:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.590704  4172 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: No bootstrap required, opened a new log
00:28:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.593811  4172 raft_consensus.cc:359] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.593969  4172 raft_consensus.cc:385] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.594015  4172 raft_consensus.cc:740] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f4b9ee514d9d4ee688da0c81e1a05f3f, State: Initialized, Role: FOLLOWER
00:28:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.594448  4172 consensus_queue.cc:260] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.594985  4282 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.595326  4282 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [sys.catalog]: This master's current role is: FOLLOWER
00:28:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.595587  4172 sys_catalog.cc:565] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [sys.catalog]: configured and running, proceeding with master startup.
00:28:05.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.601886  4293 catalog_manager.cc:1593] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:05.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.602226  4293 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.603091  4102 master_runner.cc:264] No leader master found from master f4b9ee514d9d4ee688da0c81e1a05f3f
00:28:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.603148  4102 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:28:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.638252  4255 raft_consensus.cc:493] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.638424  4255 raft_consensus.cc:515] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.639482  4255 leader_election.cc:290] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f4b9ee514d9d4ee688da0c81e1a05f3f (127.3.237.189:40855), 49d83bbc22074419af110d15665c1974 (127.3.237.188:36829)
00:28:05.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.641119  4146 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48fcb917bd34a83ad007372c20f61e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" is_pre_election: true
00:28:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.641395  4146 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c48fcb917bd34a83ad007372c20f61e1 in term 0.
00:28:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.641855  4051 leader_election.cc:304] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c48fcb917bd34a83ad007372c20f61e1, f4b9ee514d9d4ee688da0c81e1a05f3f; no voters: 
00:28:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.642391  4218 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48fcb917bd34a83ad007372c20f61e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49d83bbc22074419af110d15665c1974" is_pre_election: true
00:28:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.642621  4218 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c48fcb917bd34a83ad007372c20f61e1 in term 0.
00:28:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.643887  4255 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Leader pre-election won for term 1
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.643958  4255 raft_consensus.cc:493] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.644001  4255 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 0 FOLLOWER]: Advancing to term 1
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.645134  4255 raft_consensus.cc:515] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.645478  4255 leader_election.cc:290] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [CANDIDATE]: Term 1 election: Requested vote from peers f4b9ee514d9d4ee688da0c81e1a05f3f (127.3.237.189:40855), 49d83bbc22074419af110d15665c1974 (127.3.237.188:36829)
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.646775  4146 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48fcb917bd34a83ad007372c20f61e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f"
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.646873  4146 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [term 0 FOLLOWER]: Advancing to term 1
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.647950  4146 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c48fcb917bd34a83ad007372c20f61e1 in term 1.
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.648308  4051 leader_election.cc:304] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c48fcb917bd34a83ad007372c20f61e1, f4b9ee514d9d4ee688da0c81e1a05f3f; no voters: 
00:28:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.648681  4218 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48fcb917bd34a83ad007372c20f61e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49d83bbc22074419af110d15665c1974"
00:28:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.648777  4218 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [term 0 FOLLOWER]: Advancing to term 1
00:28:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.650429  4218 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c48fcb917bd34a83ad007372c20f61e1 in term 1.
00:28:05.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.656363  4255 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 1 FOLLOWER]: Leader election won for term 1
00:28:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.656551  4255 raft_consensus.cc:697] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [term 1 LEADER]: Becoming Leader. State: Replica: c48fcb917bd34a83ad007372c20f61e1, State: Running, Role: LEADER
00:28:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.656893  4255 consensus_queue.cc:237] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } }
00:28:05.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.659255  4255 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c48fcb917bd34a83ad007372c20f61e1. Latest consensus state: current_term: 1 leader_uuid: "c48fcb917bd34a83ad007372c20f61e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.659348  4255 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: This master's current role is: LEADER
00:28:05.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.664139  4297 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
00:28:05.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.665512  4297 catalog_manager.cc:1519] Initializing Kudu cluster ID...
00:28:05.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.667452  4053 tablet.cc:2406] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:05.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.687395  4218 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [term 1 FOLLOWER]: Refusing update from remote peer c48fcb917bd34a83ad007372c20f61e1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:28:05.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.689395  4255 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:28:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.690409  4146 raft_consensus.cc:1275] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [term 1 FOLLOWER]: Refusing update from remote peer c48fcb917bd34a83ad007372c20f61e1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:28:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.691449  4255 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:28:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.695731  4282 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [sys.catalog]: SysCatalogTable state changed. Reason: New leader c48fcb917bd34a83ad007372c20f61e1. Latest consensus state: current_term: 1 leader_uuid: "c48fcb917bd34a83ad007372c20f61e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.696045  4282 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [sys.catalog]: This master's current role is: FOLLOWER
00:28:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.698148  4256 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c48fcb917bd34a83ad007372c20f61e1. Latest consensus state: current_term: 1 leader_uuid: "c48fcb917bd34a83ad007372c20f61e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.698243  4256 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [sys.catalog]: This master's current role is: FOLLOWER
00:28:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.702263  4295 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c48fcb917bd34a83ad007372c20f61e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.702373  4295 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: This master's current role is: LEADER
00:28:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.703733  4282 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c48fcb917bd34a83ad007372c20f61e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.703868  4282 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f [sys.catalog]: This master's current role is: FOLLOWER
00:28:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.704015  4299 mvcc.cc:204] Tried to move back new op lower bound from 7305095109356756992 to 7305095109256331264. Current Snapshot: MvccSnapshot[applied={T|T < 7305095109356756992}]
00:28:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.705348  4255 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c48fcb917bd34a83ad007372c20f61e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.705446  4255 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1 [sys.catalog]: This master's current role is: LEADER
00:28:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.706951  4256 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c48fcb917bd34a83ad007372c20f61e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c48fcb917bd34a83ad007372c20f61e1" member_type: VOTER last_known_addr { host: "127.3.237.190" port: 45669 } } peers { permanent_uuid: "f4b9ee514d9d4ee688da0c81e1a05f3f" member_type: VOTER last_known_addr { host: "127.3.237.189" port: 40855 } } peers { permanent_uuid: "49d83bbc22074419af110d15665c1974" member_type: VOTER last_known_addr { host: "127.3.237.188" port: 36829 } } }
00:28:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.707051  4256 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974 [sys.catalog]: This master's current role is: FOLLOWER
00:28:05.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.708117  4301 mvcc.cc:204] Tried to move back new op lower bound from 7305095109356756992 to 7305095109256331264. Current Snapshot: MvccSnapshot[applied={T|T < 7305095109356756992}]
00:28:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.711339  4297 catalog_manager.cc:1382] Generated new cluster ID: 0cf0b0400be844bc85c4c74259d69816
00:28:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.711404  4297 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
00:28:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.726274  4297 catalog_manager.cc:1405] Generated new certificate authority record
00:28:05.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.727717  4297 catalog_manager.cc:1539] Loading token signing keys...
00:28:05.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.745796  4297 catalog_manager.cc:6069] T 00000000000000000000000000000000 P c48fcb917bd34a83ad007372c20f61e1: Generated new TSK 0
00:28:05.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.746658  4297 catalog_manager.cc:1549] Initializing in-progress tserver states...
00:28:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.754257  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:0
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.882932  4309 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.883147  4309 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.883206  4309 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.886590  4309 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.886734  4309 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.129
00:28:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.890542  4309 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:0
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4309
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:05.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:05.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:05.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.891911  4309 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:05.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.893337  4309 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.899749  4315 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.899742  4317 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:05.899742  4314 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.900295  4309 server_base.cc:1061] running on GCE node
00:28:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.900877  4309 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.901441  4309 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:05.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.902621  4309 hybrid_clock.cc:648] HybridClock initialized: now 1783470485902602 us; error 26 us; skew 500 ppm
00:28:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.904615  4309 webserver.cc:534] Webserver started at http://127.3.237.129:32913/ using document root <none> and password file <none>
00:28:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.905223  4309 fs_manager.cc:362] Metadata directory not provided
00:28:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.905297  4309 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.905490  4309 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:28:05.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.907385  4309 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/instance:
00:28:05.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:05.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.907939  4309 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal/instance:
00:28:05.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:05.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.911751  4309 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.003s	sys 0.000s
00:28:05.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.914059  4323 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.915166  4309 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.915273  4309 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.915355  4309 fs_report.cc:389] FS layout report
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/data
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:05.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.930518  4309 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:05.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.930999  4309 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:05.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.931124  4309 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:05.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.931545  4309 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.932533  4309 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:28:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.932608  4309 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:28:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.932662  4309 ts_tablet_manager.cc:616] Registered 0 tablets
00:28:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.932687  4309 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:28:05.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.941491  4309 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.129:43753
00:28:05.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.941536  4436 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.129:43753 every 8 connection(s)
00:28:05.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.942337  4309 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:05.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.947930  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4309
00:28:05.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.948040  4022 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal/instance
00:28:05.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.951015  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:05.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:0
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.952049  4438 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.952251  4438 heartbeater.cc:461] Registering TS with master...
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.952811  4438 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.953557  4439 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.953629  4439 heartbeater.cc:461] Registering TS with master...
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.953809  4439 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.953927  4437 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.953984  4437 heartbeater.cc:461] Registering TS with master...
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.954152  4437 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.954499  4064 ts_manager.cc:194] Registered new tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.954795  4136 ts_manager.cc:194] Registered new tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.955096  4208 ts_manager.cc:194] Registered new tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:05.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:05.956231  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.129:56157
00:28:06.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.052611  4444 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.052785  4444 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.052826  4444 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.056017  4444 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.056105  4444 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.130
00:28:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.059765  4444 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:0
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4444
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.060806  4444 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.061769  4444 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:06.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.067539  4452 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.067589  4450 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.067584  4449 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.067891  4444 server_base.cc:1061] running on GCE node
00:28:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.068334  4444 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.068868  4444 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:06.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.070046  4444 hybrid_clock.cc:648] HybridClock initialized: now 1783470486070025 us; error 30 us; skew 500 ppm
00:28:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.071645  4444 webserver.cc:534] Webserver started at http://127.3.237.130:46813/ using document root <none> and password file <none>
00:28:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.072233  4444 fs_manager.cc:362] Metadata directory not provided
00:28:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.072317  4444 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.072536  4444 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:28:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.074007  4444 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/instance:
00:28:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731068d234f34dc09763e08ded790c5d"
00:28:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.074534  4444 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal/instance:
00:28:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731068d234f34dc09763e08ded790c5d"
00:28:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:06.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.077593  4444 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.004s	sys 0.001s
00:28:06.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.079514  4458 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:06.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.080461  4444 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:06.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.080598  4444 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:06.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731068d234f34dc09763e08ded790c5d"
00:28:06.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:06.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.080690  4444 fs_report.cc:389] FS layout report
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/data
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.099568  4444 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.100262  4444 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.100457  4444 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.101007  4444 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.101984  4444 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:28:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.102061  4444 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:28:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.102114  4444 ts_tablet_manager.cc:616] Registered 0 tablets
00:28:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.102137  4444 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:28:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.111371  4444 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.130:41841
00:28:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.111398  4571 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.130:41841 every 8 connection(s)
00:28:06.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.112398  4444 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:06.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.118913  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4444
00:28:06.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.119012  4022 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal/instance
00:28:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.121268  4572 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.121476  4572 heartbeater.cc:461] Registering TS with master...
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.121969  4574 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.122015  4572 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.122057  4574 heartbeater.cc:461] Registering TS with master...
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.122248  4574 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.122323  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:0
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.123507  4208 ts_manager.cc:194] Registered new tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.123680  4136 ts_manager.cc:194] Registered new tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.123966  4573 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.124037  4573 heartbeater.cc:461] Registering TS with master...
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.124198  4573 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.124804  4064 ts_manager.cc:194] Registered new tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.125464  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.130:48689
00:28:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.228801  4579 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.229008  4579 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.229068  4579 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:06.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.232577  4579 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:06.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.232723  4579 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.131
00:28:06.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.236583  4579 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:06.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:0
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4579
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.237848  4579 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.239136  4579 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.245671  4585 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.245672  4584 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.246035  4579 server_base.cc:1061] running on GCE node
00:28:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.245674  4587 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.246646  4579 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:06.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.247098  4579 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:06.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.248267  4579 hybrid_clock.cc:648] HybridClock initialized: now 1783470486248262 us; error 44 us; skew 500 ppm
00:28:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.249800  4579 webserver.cc:534] Webserver started at http://127.3.237.131:41753/ using document root <none> and password file <none>
00:28:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.250263  4579 fs_manager.cc:362] Metadata directory not provided
00:28:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.250311  4579 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.250445  4579 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:28:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.251724  4579 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/instance:
00:28:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.252169  4579 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal/instance:
00:28:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.256027  4579 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.004s	sys 0.000s
00:28:06.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.258463  4593 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:06.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.259585  4579 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:06.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.259704  4579 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:06.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.259792  4579 fs_report.cc:389] FS layout report
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/data
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.280117  4579 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.280658  4579 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.280802  4579 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:06.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.281258  4579 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:06.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.282159  4579 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:28:06.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.282205  4579 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:28:06.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.282241  4579 ts_tablet_manager.cc:616] Registered 0 tablets
00:28:06.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.282256  4579 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:28:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.291389  4579 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.131:41999
00:28:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.291468  4706 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.131:41999 every 8 connection(s)
00:28:06.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.292279  4579 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.299227  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4579
00:28:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.299342  4022 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal/instance
00:28:06.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.301265  4707 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.301513  4707 heartbeater.cc:461] Registering TS with master...
00:28:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.301568  4708 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.301683  4708 heartbeater.cc:461] Registering TS with master...
00:28:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.301687  4709 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.301779  4709 heartbeater.cc:461] Registering TS with master...
00:28:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.302052  4708 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.302057  4707 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.302057  4709 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.303145  4136 ts_manager.cc:194] Registered new tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.303162  4064 ts_manager.cc:194] Registered new tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.303179  4208 ts_manager.cc:194] Registered new tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.303843  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.131:37567
00:28:06.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.307003  4022 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
00:28:06.308 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:28:06.308 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

00:28:06.316 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.3.237.190"
      port: 45669
    }
    bound_http_address {
      host: "127.3.237.190"
      port: 33407
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.3.237.189"
      port: 40855
    }
    bound_http_address {
      host: "127.3.237.189"
      port: 46297
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.3.237.188"
      port: 36829
    }
    bound_http_address {
      host: "127.3.237.188"
      port: 46363
    }
  }
}

00:28:06.319 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

00:28:06.321 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.3.237.129"
      port: 43753
    }
    bound_http_address {
      host: "127.3.237.129"
      port: 32913
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.3.237.130"
      port: 41841
    }
    bound_http_address {
      host: "127.3.237.130"
      port: 46813
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.3.237.131"
      port: 41999
    }
    bound_http_address {
      host: "127.3.237.131"
      port: 41753
    }
  }
}

00:28:06.322 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:28:06.481 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
00:28:06.538 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
00:28:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.554371  4278 catalog_manager.cc:1294] Loaded cluster ID: 0cf0b0400be844bc85c4c74259d69816
00:28:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.554473  4278 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: loading cluster ID for follower catalog manager: success
00:28:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.556829  4278 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: acquiring CA information for follower catalog manager: success
00:28:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.558342  4278 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 49d83bbc22074419af110d15665c1974: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:28:06.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.603883  4293 catalog_manager.cc:1294] Loaded cluster ID: 0cf0b0400be844bc85c4c74259d69816
00:28:06.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.603966  4293 catalog_manager.cc:1587] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: loading cluster ID for follower catalog manager: success
00:28:06.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.605705  4293 catalog_manager.cc:1609] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: acquiring CA information for follower catalog manager: success
00:28:06.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.606772  4293 catalog_manager.cc:1637] T 00000000000000000000000000000000 P f4b9ee514d9d4ee688da0c81e1a05f3f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:28:06.688 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.237.189:40855(127.3.237.189:40855)] connecting to peer
00:28:06.746 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.237.188:36829(127.3.237.188:36829)] connecting to peer
00:28:06.747 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.237.190:45669(127.3.237.190:45669)] connecting to peer
00:28:06.768 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.3.237.190:45669(127.3.237.190:45669)] Successfully connected to peer
00:28:06.768 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.3.237.188:36829(127.3.237.188:36829)] Successfully connected to peer
00:28:06.770 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.237.189:40855(127.3.237.189:40855)] Successfully connected to peer
00:28:06.868 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.237.189
00:28:06.868 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.237.188
00:28:06.870 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.237.190
00:28:06.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.957748  4438 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:07.083 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x2e9daaa6, L:/127.0.0.1:52034 - R:/127.3.237.190:45669] using SASL/PLAIN
00:28:07.083 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x91a8f971, L:/127.0.0.1:41800 - R:/127.3.237.188:36829] using SASL/PLAIN
00:28:07.085 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe8a152e7, L:/127.0.0.1:36616 - R:/127.3.237.189:40855] using SASL/PLAIN
00:28:07.117 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
00:28:07.118 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=465), Traces: [0ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.237.189:40855, [89ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.237.188:36829, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.237.190:45669, [440ms] Sub RPC ConnectToMaster: received response from server master-127.3.237.188:36829: OK, [440ms] Sub RPC ConnectToMaster: received response from server master-127.3.237.189:40855: OK, [441ms] Sub RPC ConnectToMaster: received response from server master-127.3.237.190:45669: OK, deferred=null) after lookup
00:28:07.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.127040  4573 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.184185  4064 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52034:
00:28:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640"
00:28:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
00:28:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:28:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
00:28:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:28:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:28:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:07.186030  4064 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
00:28:07.229 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 2a746c13a9cc48dc90999f7ba0c7aa57
00:28:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.233058  4641 tablet_service.cc:1511] Processing CreateTablet for tablet c30f074164564084aed8122794b78746 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640 [id=2a746c13a9cc48dc90999f7ba0c7aa57]), partition=RANGE (key) PARTITION UNBOUNDED
00:28:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.234561  4641 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c30f074164564084aed8122794b78746. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.247200  4737 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap starting.
00:28:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.247376  4371 tablet_service.cc:1511] Processing CreateTablet for tablet c30f074164564084aed8122794b78746 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640 [id=2a746c13a9cc48dc90999f7ba0c7aa57]), partition=RANGE (key) PARTITION UNBOUNDED
00:28:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.248780  4371 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c30f074164564084aed8122794b78746. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.250079  4737 tablet_bootstrap.cc:654] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Neither blocks nor log segments found. Creating new log.
00:28:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.250159  4506 tablet_service.cc:1511] Processing CreateTablet for tablet c30f074164564084aed8122794b78746 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640 [id=2a746c13a9cc48dc90999f7ba0c7aa57]), partition=RANGE (key) PARTITION UNBOUNDED
00:28:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.250877  4737 log.cc:826] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Log is configured to *not* fsync() on all Append() calls
00:28:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.251031  4506 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c30f074164564084aed8122794b78746. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.253255  4737 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: No bootstrap required, opened a new log
00:28:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.253433  4737 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent bootstrapping tablet: real 0.006s	user 0.005s	sys 0.000s
00:28:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.256101  4739 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap starting.
00:28:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.257409  4737 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.257632  4737 raft_consensus.cc:385] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.257732  4737 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Initialized, Role: FOLLOWER
00:28:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.258219  4737 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.259064  4737 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
00:28:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.259594  4739 tablet_bootstrap.cc:654] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Neither blocks nor log segments found. Creating new log.
00:28:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.260617  4708 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.262007  4739 log.cc:826] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Log is configured to *not* fsync() on all Append() calls
00:28:07.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.263892  4739 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: No bootstrap required, opened a new log
00:28:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.266763  4739 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent bootstrapping tablet: real 0.011s	user 0.005s	sys 0.000s
00:28:07.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.268472  4741 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap starting.
00:28:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.270150  4739 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.270517  4739 raft_consensus.cc:385] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.270732  4741 tablet_bootstrap.cc:654] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Neither blocks nor log segments found. Creating new log.
00:28:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.270731  4739 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Initialized, Role: FOLLOWER
00:28:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.271184  4739 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.271715  4741 log.cc:826] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Log is configured to *not* fsync() on all Append() calls
00:28:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.272259  4739 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
00:28:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.277720  4741 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: No bootstrap required, opened a new log
00:28:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.278234  4741 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent bootstrapping tablet: real 0.010s	user 0.005s	sys 0.000s
00:28:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.281812  4741 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.282217  4741 raft_consensus.cc:385] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.282418  4741 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 731068d234f34dc09763e08ded790c5d, State: Initialized, Role: FOLLOWER
00:28:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.283022  4741 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.284060  4741 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent starting tablet: real 0.006s	user 0.000s	sys 0.005s
00:28:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:07.294595  4710 tablet.cc:2406] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.334877  4740 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.335084  4740 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.338052  4740 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.343397  4391 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f" is_pre_election: true
00:28:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.343695  4391 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 0.
00:28:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.344496  4597 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 
00:28:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.344789  4740 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Leader pre-election won for term 1
00:28:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.344854  4740 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:28:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.344913  4740 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 0 FOLLOWER]: Advancing to term 1
00:28:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.346040  4740 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.346413  4740 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 1 election: Requested vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.347434  4391 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.347555  4391 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 0 FOLLOWER]: Advancing to term 1
00:28:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.348686  4391 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 1.
00:28:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.349048  4597 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 
00:28:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.349606  4740 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Leader election won for term 1
00:28:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.349823  4740 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 LEADER]: Becoming Leader. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Running, Role: LEADER
00:28:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.350149  4740 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.351539  4526 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d" is_pre_election: true
00:28:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.351935  4525 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d"
00:28:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.351774  4526 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 0.
00:28:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.357915  4063 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131). New cstate: current_term: 1 leader_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:07.364394  4575 tablet.cc:2406] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:07.419 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 2a746c13a9cc48dc90999f7ba0c7aa57
00:28:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:07.444973  4440 tablet.cc:2406] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:07.447 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:07.452 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299995, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:07.453 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [8ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=null) after lookup
00:28:07.461 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:07.464 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] Successfully connected to peer
00:28:07.520 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x58fc08f2, L:/127.0.0.1:33848 - R:/127.3.237.131:41999] using TOKEN/null
00:28:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.527683  4526 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 0 FOLLOWER]: Advancing to term 1
00:28:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.527916  4391 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Refusing update from remote peer 2ddcbfc125f04fd592241ffd11b0dfb7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:28:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.528622  4740 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:28:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.529141  4526 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 1 FOLLOWER]: Refusing update from remote peer 2ddcbfc125f04fd592241ffd11b0dfb7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:28:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.531909  4740 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:28:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.539314  4751 mvcc.cc:204] Tried to move back new op lower bound from 7305095116906758144 to 7305095116202020864. Current Snapshot: MvccSnapshot[applied={T|T < 7305095116906758144}]
00:28:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.540011  4752 mvcc.cc:204] Tried to move back new op lower bound from 7305095116906758144 to 7305095116202020864. Current Snapshot: MvccSnapshot[applied={T|T < 7305095116906758144}]
00:28:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.544740  4754 mvcc.cc:204] Tried to move back new op lower bound from 7305095116906758144 to 7305095116202020864. Current Snapshot: MvccSnapshot[applied={T|T < 7305095116906758144}]
00:28:07.611 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:07.612 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.131:41999
00:28:07.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.614348  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4579
00:28:07.622 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:07.624 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] handling channelInactive
00:28:07.624 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state READY due to: connection closed
00:28:07.625 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: connection closed
00:28:07.636 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:07.636 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:07.636 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test)) after lookup
00:28:07.637 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:07.639 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.639 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.640 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.656 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:07.656 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:07.656 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [34ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539))) after lookup
00:28:07.657 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:07.658 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.659 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.659 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.675 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:07.676 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:07.676 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [34ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [35ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [53ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556))) after lookup
00:28:07.677 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:07.678 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.679 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.679 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.757 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:07.757 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:07.758 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=135), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [34ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [35ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [53ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [54ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [134ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635))) after lookup
00:28:07.759 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:07.760 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.761 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.761 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.777 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:07.777 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:07.778 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=155), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [34ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [35ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [53ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [54ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [134ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [136ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [139ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [139ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [154ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433))) after lookup
00:28:07.779 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:07.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:07.780 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.073061  4764 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.073184  4764 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.073621  4765 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:09.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.073743  4765 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:09.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.074301  4764 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:09.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:09.075157  4462 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:09.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.075378  4765 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:09.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:09.075438  4327 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:09.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:09.076385  4327 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:09.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:09.076920  4462 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.078943  4391 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f" is_pre_election: true
00:28:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.078970  4526 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d" is_pre_election: true
00:28:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079070  4526 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 1.
00:28:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079074  4391 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 731068d234f34dc09763e08ded790c5d in term 1.
00:28:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079530  4326 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079530  4462 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079759  4765 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Leader pre-election won for term 2
00:28:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079851  4765 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079903  4765 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 1 FOLLOWER]: Advancing to term 2
00:28:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079914  4764 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 1 FOLLOWER]: Leader pre-election won for term 2
00:28:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079957  4764 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.079983  4764 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 1 FOLLOWER]: Advancing to term 2
00:28:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.080695  4764 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.080725  4765 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.081024  4764 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 2 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.081115  4765 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 2 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.081435  4526 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d"
00:28:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.081581  4526 raft_consensus.cc:2393] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate f209795a44804bb3aea71582dc28cf0f in current term 2: Already voted for candidate 731068d234f34dc09763e08ded790c5d in this term.
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.081820  4391 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.081971  4391 raft_consensus.cc:2393] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 731068d234f34dc09763e08ded790c5d in current term 2: Already voted for candidate f209795a44804bb3aea71582dc28cf0f in this term.
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:09.082024  4327 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.082091  4327 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7, 731068d234f34dc09763e08ded790c5d
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.082247  4765 raft_consensus.cc:2749] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:09.082599  4462 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.082664  4462 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 731068d234f34dc09763e08ded790c5d; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f
00:28:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:09.082798  4764 raft_consensus.cc:2749] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:28:09.697 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:09.697 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:09.697 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2075), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [34ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [35ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [53ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [54ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [134ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [136ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [139ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [139ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [154ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [156ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [158ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [158ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2071ms] refreshing cache from master, [2071ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2074ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433) -> (continuation of Deferred@1796053922 after retry RPC@21224789), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433) -> (continuation of Deferred@1796053922 after retry RPC@21224789))) after lookup
00:28:09.698 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:09.699 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:09.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:09.699 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:10.036 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:10.036 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:10.037 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2414), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [34ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [35ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [53ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [54ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [134ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [136ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [139ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [139ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [154ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [156ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [158ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [158ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2071ms] refreshing cache from master, [2071ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2074ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [2076ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2077ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2077ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2411ms] refreshing cache from master, [2411ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2414ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433) -> (continuation of Deferred@1796053922 after retry RPC@21224789) -> (continuation of Deferred@762136283 after retry RPC@844865366), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433) -> (continuation of Deferred@1796053922 after retry RPC@21224789) -> (continuation of Deferred@762136283 after retry RPC@844865366))) after lookup
00:28:10.037 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:10.038 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:10.038 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:10.038 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.073904  4776 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.074054  4776 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.074525  4776 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.075102  4526 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d" is_pre_election: true
00:28:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.075248  4526 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 2.
00:28:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.075618  4326 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 
00:28:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.075914  4776 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Leader pre-election won for term 3
00:28:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.076136  4776 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:28:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:11.076182  4327 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.076213  4776 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 2 FOLLOWER]: Advancing to term 3
00:28:11.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.077095  4776 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:11.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.077467  4776 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 3 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.077891  4526 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d"
00:28:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.078027  4526 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 2 FOLLOWER]: Advancing to term 3
00:28:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:11.078495  4327 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.078963  4526 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 3.
00:28:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.079274  4326 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.079440  4776 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 3 FOLLOWER]: Leader election won for term 3
00:28:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.079627  4776 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 3 LEADER]: Becoming Leader. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Running, Role: LEADER
00:28:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.079982  4776 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:11.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.082345  4064 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f reported cstate change: term changed from 1 to 3, leader changed from 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131) to f209795a44804bb3aea71582dc28cf0f (127.3.237.129). New cstate: current_term: 3 leader_uuid: "f209795a44804bb3aea71582dc28cf0f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:11.520483  4327 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f -> Peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Couldn't send request to peer 2ddcbfc125f04fd592241ffd11b0dfb7. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.535468  4526 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 3 FOLLOWER]: Refusing update from remote peer f209795a44804bb3aea71582dc28cf0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
00:28:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.536048  4776 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
00:28:12.916 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:12.919 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:12.920 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=5297), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [13ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [34ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [35ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [53ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [54ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [132ms] refreshing cache from master, [132ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [134ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [136ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [139ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [139ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [154ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [156ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [158ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [158ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2071ms] refreshing cache from master, [2071ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2074ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [2076ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2077ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2077ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2411ms] refreshing cache from master, [2411ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2414ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [2415ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2416ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2417ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [5291ms] refreshing cache from master, [5291ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [5294ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@706715435(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@101641a4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433) -> (continuation of Deferred@1796053922 after retry RPC@21224789) -> (continuation of Deferred@762136283 after retry RPC@844865366) -> (continuation of Deferred@1835219462 after retry RPC@2066783290), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5de8530f -> wakeup thread Time-limited test -> (continuation of Deferred@858069587 after retry RPC@1930324539) -> (continuation of Deferred@20263381 after retry RPC@1318221556) -> (continuation of Deferred@1138327960 after retry RPC@1184080635) -> (continuation of Deferred@596643377 after retry RPC@329197433) -> (continuation of Deferred@1796053922 after retry RPC@21224789) -> (continuation of Deferred@762136283 after retry RPC@844865366) -> (continuation of Deferred@1835219462 after retry RPC@2066783290))) after lookup
00:28:12.920 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:12.921 [DEBUG - kudu-nio-3] (Connection.java:757) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] Successfully connected to peer
00:28:12.939 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x69f6bb3b, L:/127.0.0.1:49404 - R:/127.3.237.129:43753] using TOKEN/null
00:28:12.961 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.131:41999
00:28:12.962 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:12.963132  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.061585  4807 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.061764  4807 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.061805  4807 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.064937  4807 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.065037  4807 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.131
00:28:13.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.068712  4807 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4807
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.069654  4807 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.070974  4807 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.078888  4812 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.078938  4813 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.079262  4807 server_base.cc:1061] running on GCE node
00:28:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.078902  4815 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.080067  4807 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.080655  4807 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:13.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.081830  4807 hybrid_clock.cc:648] HybridClock initialized: now 1783470493081815 us; error 32 us; skew 500 ppm
00:28:13.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.083705  4807 webserver.cc:534] Webserver started at http://127.3.237.131:41753/ using document root <none> and password file <none>
00:28:13.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.084233  4807 fs_manager.cc:362] Metadata directory not provided
00:28:13.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.084298  4807 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:13.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.087495  4807 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
00:28:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.089252  4821 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.090181  4807 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
00:28:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.090301  4807 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.090720  4807 fs_report.cc:389] FS layout report
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/data
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.108103  4807 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.108752  4807 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.108966  4807 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.109555  4807 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.110725  4828 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.113852  4807 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.113914  4807 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.113966  4807 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.115563  4807 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.115610  4807 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
00:28:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.116014  4828 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap starting.
00:28:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.126013  4807 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.131:41999
00:28:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.126082  4935 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.131:41999 every 8 connection(s)
00:28:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.127192  4807 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:13.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.129029  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4807
00:28:13.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:13.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.134326  4828 log.cc:826] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Log is configured to *not* fsync() on all Append() calls
00:28:13.139 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:13.140 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.129:43753
00:28:13.140 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.141155  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4309
00:28:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.145437  4828 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.145766  4938 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.146168  4938 heartbeater.cc:461] Registering TS with master...
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.146325  4828 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap complete.
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.146692  4938 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.147686  4828 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent bootstrapping tablet: real 0.032s	user 0.022s	sys 0.004s
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.145771  4936 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.147925  4936 heartbeater.cc:461] Registering TS with master...
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.148100  4936 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.148487  4937 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.148556  4937 heartbeater.cc:461] Registering TS with master...
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.148720  4937 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.149240  4208 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:13.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.150204  4064 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.150661  4136 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.150553  4828 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.150952  4828 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Initialized, Role: FOLLOWER
00:28:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.151160  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.131:39085
00:28:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.151407  4828 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.152256  4828 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
00:28:13.153 [DEBUG - kudu-nio-3] (Connection.java:239) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] handling channelInactive
00:28:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.153187  4937 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:13.153 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state READY due to: connection closed
00:28:13.153 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: connection closed
00:28:13.176 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:13.176 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:13.177 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [3ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [26ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@488415567(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@62b22f9a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@18369c8f -> wakeup thread Time-limited test)) after lookup
00:28:13.177 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:13.178 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.179 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.179 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.195 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:13.196 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:13.196 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [3ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [26ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [27ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [45ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@488415567(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@62b22f9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@18369c8f -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981))) after lookup
00:28:13.197 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:13.197 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.198 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.198 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.216 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:13.216 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:13.217 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [3ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [26ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [27ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [45ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [46ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [48ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [48ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [66ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@488415567(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@62b22f9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@18369c8f -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179))) after lookup
00:28:13.217 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:13.218 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.218 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.218 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:13.357 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:13.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=207), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [3ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [26ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [27ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [45ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [46ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [48ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [48ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [66ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [67ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [68ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [68ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [207ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@488415567(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@62b22f9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@18369c8f -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728))) after lookup
00:28:13.358 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:13.359 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.359 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.360 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:13.798 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:13.798 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=647), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [3ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [26ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [27ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [45ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [46ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [48ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [48ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [66ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [67ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [68ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [68ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [207ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [208ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [209ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [210ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [644ms] refreshing cache from master, [644ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [647ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@488415567(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@62b22f9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728) -> (continuation of Deferred@1669337062 after retry RPC@2124703120), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@18369c8f -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728) -> (continuation of Deferred@1669337062 after retry RPC@2124703120))) after lookup
00:28:13.799 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:13.800 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:13.800 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:14.296 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:14.297 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:14.297 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1147), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [3ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [26ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [27ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [45ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [46ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [48ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [48ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [66ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [67ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [68ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [68ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [207ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [208ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [209ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [210ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [644ms] refreshing cache from master, [644ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [647ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [649ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [650ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [650ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [1143ms] refreshing cache from master, [1143ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [1146ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@488415567(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@62b22f9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728) -> (continuation of Deferred@1669337062 after retry RPC@2124703120) -> (continuation of Deferred@1011017562 after retry RPC@1637322151), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@18369c8f -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728) -> (continuation of Deferred@1669337062 after retry RPC@2124703120) -> (continuation of Deferred@1011017562 after retry RPC@1637322151))) after lookup
00:28:14.298 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:14.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:14.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:14.299 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:14.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.338243  4949 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:14.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.338467  4949 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:14.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.339708  4949 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:14.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:14.340618  4825 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:14.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:14.341620  4825 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:14.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.343955  4526 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d" is_pre_election: true
00:28:14.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.344537  4824 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7; no voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f
00:28:14.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.344750  4949 raft_consensus.cc:2749] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
00:28:14.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.460139  4954 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 3 FOLLOWER]: Starting pre-election (detected failure of leader f209795a44804bb3aea71582dc28cf0f)
00:28:14.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.460278  4954 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:14.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.460706  4954 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:14.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:14.461928  4462 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111) [suppressed 1 similar messages]
00:28:14.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:14.462803  4462 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.465935  4890 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" is_pre_election: true
00:28:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.466140  4890 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 731068d234f34dc09763e08ded790c5d in term 1.
00:28:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.466715  4462 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, 731068d234f34dc09763e08ded790c5d; no voters: f209795a44804bb3aea71582dc28cf0f
00:28:14.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.466899  4954 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 3 FOLLOWER]: Leader pre-election won for term 4
00:28:14.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.466970  4954 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 3 FOLLOWER]: Starting leader election (detected failure of leader f209795a44804bb3aea71582dc28cf0f)
00:28:14.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.467007  4954 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 3 FOLLOWER]: Advancing to term 4
00:28:14.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.467790  4954 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:14.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.468147  4954 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 4 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:14.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.468469  4890 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:14.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.468624  4890 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 1 FOLLOWER]: Advancing to term 4
00:28:14.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:14.469141  4462 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.470300  4890 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 731068d234f34dc09763e08ded790c5d in term 4.
00:28:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.470619  4462 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, 731068d234f34dc09763e08ded790c5d; no voters: f209795a44804bb3aea71582dc28cf0f
00:28:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.470829  4954 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Leader election won for term 4
00:28:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.471024  4954 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 LEADER]: Becoming Leader. State: Replica: 731068d234f34dc09763e08ded790c5d, State: Running, Role: LEADER
00:28:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.471311  4954 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.473588  4064 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d reported cstate change: term changed from 3 to 4, leader changed from f209795a44804bb3aea71582dc28cf0f (127.3.237.129) to 731068d234f34dc09763e08ded790c5d (127.3.237.130). New cstate: current_term: 4 leader_uuid: "731068d234f34dc09763e08ded790c5d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: HEALTHY } } }
00:28:14.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.953540  4890 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 4 FOLLOWER]: Refusing update from remote peer 731068d234f34dc09763e08ded790c5d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
00:28:14.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.954216  4954 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
00:28:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:14.962078  4462 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d -> Peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Couldn't send request to peer f209795a44804bb3aea71582dc28cf0f. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:17.414036  4462 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d -> Peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Couldn't send request to peer f209795a44804bb3aea71582dc28cf0f. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:17.997 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:17.997 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:17.999 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=4848), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [3ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [26ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [27ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [45ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [46ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [48ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [48ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [66ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [67ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [68ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [68ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [207ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [208ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [209ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [210ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [644ms] refreshing cache from master, [644ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [647ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [649ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [650ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [650ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [1143ms] refreshing cache from master, [1143ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [1146ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [1148ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [1149ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [1149ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [4843ms] refreshing cache from master, [4843ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [4846ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@488415567(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@62b22f9a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728) -> (continuation of Deferred@1669337062 after retry RPC@2124703120) -> (continuation of Deferred@1011017562 after retry RPC@1637322151) -> (continuation of Deferred@298200554 after retry RPC@1289838784), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@18369c8f -> wakeup thread Time-limited test -> (continuation of Deferred@572804042 after retry RPC@249194981) -> (continuation of Deferred@9780109 after retry RPC@1566437179) -> (continuation of Deferred@1449792081 after retry RPC@1694243728) -> (continuation of Deferred@1669337062 after retry RPC@2124703120) -> (continuation of Deferred@1011017562 after retry RPC@1637322151) -> (continuation of Deferred@298200554 after retry RPC@1289838784))) after lookup
00:28:18.000 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:18.001 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] Successfully connected to peer
00:28:18.016 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x363842ca, L:/127.0.0.1:35512 - R:/127.3.237.130:41841] using TOKEN/null
00:28:18.035 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.129:43753
00:28:18.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:18.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.038625  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:18.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:18.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:43753
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32913
00:28:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:18.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.143198  4996 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:18.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.143388  4996 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:18.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.143433  4996 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:18.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.146719  4996 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:18.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.146818  4996 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.129
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.150871  4996 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:43753
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32913
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4996
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.152025  4996 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:18.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.153333  4996 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:18.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.159729  5001 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:18.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.159847  4996 server_base.cc:1061] running on GCE node
00:28:18.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.159737  5002 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:18.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.159737  5004 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:18.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.160828  4996 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:18.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.161399  4996 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:18.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.162585  4996 hybrid_clock.cc:648] HybridClock initialized: now 1783470498162561 us; error 35 us; skew 500 ppm
00:28:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.164359  4996 webserver.cc:534] Webserver started at http://127.3.237.129:32913/ using document root <none> and password file <none>
00:28:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.164933  4996 fs_manager.cc:362] Metadata directory not provided
00:28:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.165091  4996 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:18.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.168313  4996 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.001s
00:28:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.170183  5010 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.171101  4996 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.171224  4996 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.171653  4996 fs_report.cc:389] FS layout report
00:28:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/data
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.188109  4996 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.188799  4996 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.189003  4996 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.189582  4996 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:18.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.190843  5017 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.193991  4996 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.194067  4996 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.194144  4996 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:18.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.195525  4996 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:18.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.195590  4996 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
00:28:18.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.195950  5017 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap starting.
00:28:18.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.206941  4996 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.129:43753
00:28:18.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.206991  5124 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.129:43753 every 8 connection(s)
00:28:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.207939  4996 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.212126  5017 log.cc:826] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Log is configured to *not* fsync() on all Append() calls
00:28:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.217152  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 4996
00:28:18.217 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.222095  5126 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.222106  5127 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.222347  5126 heartbeater.cc:461] Registering TS with master...
00:28:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.222345  5127 heartbeater.cc:461] Registering TS with master...
00:28:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.222932  5127 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:18.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.223408  5017 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:18.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.222932  5126 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:18.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.223801  5017 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap complete.
00:28:18.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.224253  5125 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:18.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.224335  5125 heartbeater.cc:461] Registering TS with master...
00:28:18.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.224532  5125 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:18.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.225198  5017 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent bootstrapping tablet: real 0.030s	user 0.019s	sys 0.006s
00:28:18.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.225306  4135 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.225890  4207 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.226624  4063 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.227576  4063 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.129:60427
00:28:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.227684  5017 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.228070  5017 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Initialized, Role: FOLLOWER
00:28:18.228 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.228518  5017 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.229349  5017 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
00:28:18.229 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.130:41841
00:28:18.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.229717  5126 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:18.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:18.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.230444  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4444
00:28:18.238 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] handling channelInactive
00:28:18.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:18.238 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state READY due to: connection closed
00:28:18.239 [DEBUG - Time-limited test] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:18.240 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.240 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.240 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.257 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:18.257 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:18.257 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [1ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [17ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@659267344(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@456bec7c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c6f4a4b -> wakeup thread Time-limited test)) after lookup
00:28:18.258 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:18.259 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.259 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.259 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.276 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:18.276 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:18.276 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [1ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [17ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [20ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [20ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [36ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@659267344(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@456bec7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c6f4a4b -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889))) after lookup
00:28:18.277 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:18.277 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.277 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.278 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.296 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:18.297 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:18.297 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [1ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [17ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [20ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [20ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [36ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [37ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [39ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [39ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [57ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@659267344(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@456bec7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c6f4a4b -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817))) after lookup
00:28:18.297 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:18.298 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.298 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.299 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.377 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:18.377 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:18.377 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=138), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [1ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [17ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [20ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [20ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [36ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [37ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [39ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [39ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [57ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [58ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [59ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [60ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [134ms] refreshing cache from master, [135ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [137ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@659267344(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@456bec7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817) -> (continuation of Deferred@778415823 after retry RPC@649672666), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c6f4a4b -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817) -> (continuation of Deferred@778415823 after retry RPC@649672666))) after lookup
00:28:18.378 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:18.379 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.379 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.379 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.896 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:18.897 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:18.897 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=658), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [1ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [17ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [20ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [20ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [36ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [37ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [39ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [39ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [57ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [58ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [59ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [60ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [134ms] refreshing cache from master, [135ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [137ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [138ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [140ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [140ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [654ms] refreshing cache from master, [654ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [657ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@659267344(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@456bec7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817) -> (continuation of Deferred@778415823 after retry RPC@649672666) -> (continuation of Deferred@1595040597 after retry RPC@821729418), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c6f4a4b -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817) -> (continuation of Deferred@778415823 after retry RPC@649672666) -> (continuation of Deferred@1595040597 after retry RPC@821729418))) after lookup
00:28:18.898 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:18.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:18.900 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.535080  5138 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 731068d234f34dc09763e08ded790c5d)
00:28:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.535238  5138 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.535751  5138 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:19.536410  4824 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:19.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:19.537235  4824 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.540788  5079 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f" is_pre_election: true
00:28:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.541023  5079 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 3.
00:28:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.541590  4825 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 731068d234f34dc09763e08ded790c5d
00:28:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.541764  5138 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 4 FOLLOWER]: Leader pre-election won for term 5
00:28:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.541841  5138 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 731068d234f34dc09763e08ded790c5d)
00:28:19.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.541887  5138 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 4 FOLLOWER]: Advancing to term 5
00:28:19.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.542703  5138 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.543093  5138 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 5 election: Requested vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.543385  5079 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.543531  5079 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 3 FOLLOWER]: Advancing to term 5
00:28:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:19.544171  4824 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.545250  5079 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 5.
00:28:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.545552  4825 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 731068d234f34dc09763e08ded790c5d
00:28:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.545723  5138 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 5 FOLLOWER]: Leader election won for term 5
00:28:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.545974  5138 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 5 LEADER]: Becoming Leader. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Running, Role: LEADER
00:28:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.546283  5138 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:19.548857  4063 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 reported cstate change: term changed from 4 to 5, leader changed from 731068d234f34dc09763e08ded790c5d (127.3.237.130) to 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131). New cstate: current_term: 5 leader_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:20.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.118772  5079 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 5 FOLLOWER]: Refusing update from remote peer 2ddcbfc125f04fd592241ffd11b0dfb7: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
00:28:20.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.119460  5155 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
00:28:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.127782  4824 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 -> Peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Couldn't send request to peer 731068d234f34dc09763e08ded790c5d. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:20.796 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:20.797 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:20.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2558), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [1ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [17ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [20ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [20ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [36ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [37ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [39ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [39ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [57ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [58ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [59ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [60ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [134ms] refreshing cache from master, [135ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [137ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [138ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [140ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [140ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [654ms] refreshing cache from master, [654ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [657ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [659ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [661ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [661ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2554ms] refreshing cache from master, [2554ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2557ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@659267344(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@456bec7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817) -> (continuation of Deferred@778415823 after retry RPC@649672666) -> (continuation of Deferred@1595040597 after retry RPC@821729418) -> (continuation of Deferred@2104099718 after retry RPC@431028841), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c6f4a4b -> wakeup thread Time-limited test -> (continuation of Deferred@1235660796 after retry RPC@1866303889) -> (continuation of Deferred@765895973 after retry RPC@785698817) -> (continuation of Deferred@778415823 after retry RPC@649672666) -> (continuation of Deferred@1595040597 after retry RPC@821729418) -> (continuation of Deferred@2104099718 after retry RPC@431028841))) after lookup
00:28:20.798 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:20.799 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] Successfully connected to peer
00:28:20.819 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x5414a001, L:/127.0.0.1:42286 - R:/127.3.237.131:41999] using TOKEN/null
00:28:20.838 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.130:41841
00:28:20.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.839542  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:41841
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46813
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.941764  5176 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.941931  5176 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.941973  5176 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.945492  5176 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.945585  5176 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.130
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.949330  5176 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:41841
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46813
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5176
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.950412  5176 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.951665  5176 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.957842  5181 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.957808  5182 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.957885  5176 server_base.cc:1061] running on GCE node
00:28:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.957983  5184 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.958602  5176 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.959096  5176 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.960294  5176 hybrid_clock.cc:648] HybridClock initialized: now 1783470500960272 us; error 32 us; skew 500 ppm
00:28:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.962143  5176 webserver.cc:534] Webserver started at http://127.3.237.130:46813/ using document root <none> and password file <none>
00:28:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.962736  5176 fs_manager.cc:362] Metadata directory not provided
00:28:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.962858  5176 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:20.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.966300  5176 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.001s	sys 0.000s
00:28:20.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.968240  5190 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.969309  5176 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.969456  5176 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731068d234f34dc09763e08ded790c5d"
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.969873  5176 fs_report.cc:389] FS layout report
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/data
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.984881  5176 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.985602  5176 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.985809  5176 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.986454  5176 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.987934  5197 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.991003  5176 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.991078  5176 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.991145  5176 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.992580  5176 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.992650  5176 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
00:28:20.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.993052  5197 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap starting.
00:28:21.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.004154  5176 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.130:41841
00:28:21.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.004179  5304 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.130:41841 every 8 connection(s)
00:28:21.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.005172  5176 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:21.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.005679  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 5176
00:28:21.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.009744  5197 log.cc:826] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Log is configured to *not* fsync() on all Append() calls
00:28:21.017 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.020011  5307 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.020257  5307 heartbeater.cc:461] Registering TS with master...
00:28:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.020864  5307 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.021986  5306 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.022102  5306 heartbeater.cc:461] Registering TS with master...
00:28:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.022316  5306 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.022805  4135 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.023941  4063 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.024931  4063 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.130:37641
00:28:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.025884  5305 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.025964  5305 heartbeater.cc:461] Registering TS with master...
00:28:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.026340  5305 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.026872  5197 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.027208  4208 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.027266  5197 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap complete.
00:28:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.028681  5197 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent bootstrapping tablet: real 0.036s	user 0.019s	sys 0.012s
00:28:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.031213  5197 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:21.031 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.131:41999
00:28:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.031656  5197 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 731068d234f34dc09763e08ded790c5d, State: Initialized, Role: FOLLOWER
00:28:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.032160  5197 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:21.032 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.032605  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4807
00:28:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.033581  5306 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:21.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.033746  5197 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
00:28:21.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:21.040 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] handling channelInactive
00:28:21.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state READY due to: connection closed
00:28:21.041 [DEBUG - Time-limited test] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.042 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.042 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.043 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.056 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:21.056 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:21.056 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test)) after lookup
00:28:21.057 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.058 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.058 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.058 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.076 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:21.076 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:21.076 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [17ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [17ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364))) after lookup
00:28:21.077 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.078 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.078 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.078 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.096 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:21.096 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:21.096 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [17ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [17ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [54ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774))) after lookup
00:28:21.097 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.098 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.098 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.098 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.236 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:21.236 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:21.237 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=196), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [17ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [17ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [54ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [195ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121))) after lookup
00:28:21.237 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.238 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.238 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.239 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.316 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:21.316 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:21.317 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=276), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [17ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [17ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [54ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [195ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [196ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [198ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [198ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [272ms] refreshing cache from master, [272ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [275ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534))) after lookup
00:28:21.317 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.319 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.319 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.319 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.496 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:21.496 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:21.497 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=455), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [17ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [17ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [54ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [195ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [196ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [198ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [198ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [272ms] refreshing cache from master, [272ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [275ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [276ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [278ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [278ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [452ms] refreshing cache from master, [452ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [454ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534) -> (continuation of Deferred@2003990397 after retry RPC@335729572), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534) -> (continuation of Deferred@2003990397 after retry RPC@335729572))) after lookup
00:28:21.497 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.498 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.596 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:21.596 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:21.597 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=556), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [17ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [17ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [54ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [195ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [196ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [198ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [198ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [272ms] refreshing cache from master, [272ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [275ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [276ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [278ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [278ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [452ms] refreshing cache from master, [452ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [454ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [456ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [457ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [457ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [552ms] refreshing cache from master, [552ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [555ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534) -> (continuation of Deferred@2003990397 after retry RPC@335729572) -> (continuation of Deferred@1965707341 after retry RPC@2008494783), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534) -> (continuation of Deferred@2003990397 after retry RPC@335729572) -> (continuation of Deferred@1965707341 after retry RPC@2008494783))) after lookup
00:28:21.597 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:21.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:21.599 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.207448  5318 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.207656  5318 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:22.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.208885  5318 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:22.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.209683  5194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:22.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.210575  5194 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.212883  5079 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f" is_pre_election: true
00:28:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.213377  5194 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 731068d234f34dc09763e08ded790c5d; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f
00:28:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.213634  5318 raft_consensus.cc:2749] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
00:28:22.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.338541  5323 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:22.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.338665  5323 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:22.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.339672  5323 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:22.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.340322  5014 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.341239  5014 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:22.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.344638  5259 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d" is_pre_election: true
00:28:22.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.344877  5259 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 4.
00:28:22.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.345623  5013 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:22.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.345894  5323 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 5 FOLLOWER]: Leader pre-election won for term 6
00:28:22.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.345961  5323 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 5 FOLLOWER]: Starting leader election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:22.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.345999  5323 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 5 FOLLOWER]: Advancing to term 6
00:28:22.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.346951  5323 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:22.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.347333  5323 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 6 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:22.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.347709  5259 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d"
00:28:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.347903  5259 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 4 FOLLOWER]: Advancing to term 6
00:28:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.348408  5014 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:22.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.349779  5259 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 6.
00:28:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.350077  5013 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.350301  5323 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 6 FOLLOWER]: Leader election won for term 6
00:28:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.350543  5323 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 6 LEADER]: Becoming Leader. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Running, Role: LEADER
00:28:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.350833  5323 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:22.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.356258  4063 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f reported cstate change: term changed from 5 to 6, leader changed from 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131) to f209795a44804bb3aea71582dc28cf0f (127.3.237.129). New cstate: current_term: 6 leader_uuid: "f209795a44804bb3aea71582dc28cf0f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:22.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.833791  5014 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f -> Peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Couldn't send request to peer 2ddcbfc125f04fd592241ffd11b0dfb7. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.966104  5259 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 6 FOLLOWER]: Refusing update from remote peer f209795a44804bb3aea71582dc28cf0f: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 17. (index mismatch)
00:28:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.966805  5323 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
00:28:23.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:23.337 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:23.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=16, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2296), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [14ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [15ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [17ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [17ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [54ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [57ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [57ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [195ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [196ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [198ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [198ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [272ms] refreshing cache from master, [272ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [275ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [276ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [278ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [278ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [452ms] refreshing cache from master, [452ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [454ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [456ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [457ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [457ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [552ms] refreshing cache from master, [552ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [555ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [556ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [558ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [558ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2292ms] refreshing cache from master, [2293ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2295ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@420665582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@48cf96a7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534) -> (continuation of Deferred@2003990397 after retry RPC@335729572) -> (continuation of Deferred@1965707341 after retry RPC@2008494783) -> (continuation of Deferred@1023647134 after retry RPC@1945068107), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6ebb32 -> wakeup thread Time-limited test -> (continuation of Deferred@59265628 after retry RPC@405069364) -> (continuation of Deferred@95141793 after retry RPC@1002439774) -> (continuation of Deferred@1695915169 after retry RPC@1590462121) -> (continuation of Deferred@127463050 after retry RPC@1029429534) -> (continuation of Deferred@2003990397 after retry RPC@335729572) -> (continuation of Deferred@1965707341 after retry RPC@2008494783) -> (continuation of Deferred@1023647134 after retry RPC@1945068107))) after lookup
00:28:23.338 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:23.339 [DEBUG - kudu-nio-2] (Connection.java:757) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] Successfully connected to peer
00:28:23.357 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x2d79927f, L:/127.0.0.1:40228 - R:/127.3.237.129:43753] using TOKEN/null
00:28:23.377 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.131:41999
00:28:23.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.378041  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.478806  5351 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.479038  5351 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.479099  5351 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.482411  5351 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.482549  5351 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.131
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.486171  5351 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5351
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.487380  5351 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:23.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.488498  5351 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:23.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.494535  5357 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.494545  5356 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.494545  5359 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.494777  5351 server_base.cc:1061] running on GCE node
00:28:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.495391  5351 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:23.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.495939  5351 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.497098  5351 hybrid_clock.cc:648] HybridClock initialized: now 1783470503497085 us; error 26 us; skew 500 ppm
00:28:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.498817  5351 webserver.cc:534] Webserver started at http://127.3.237.131:41753/ using document root <none> and password file <none>
00:28:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.499356  5351 fs_manager.cc:362] Metadata directory not provided
00:28:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.499439  5351 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:23.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.502488  5351 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
00:28:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.504272  5365 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.505128  5351 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.505249  5351 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.505633  5351 fs_report.cc:389] FS layout report
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/data
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:23.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.527058  5351 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:23.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.527724  5351 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.527954  5351 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.528530  5351 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:23.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.529738  5372 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.532770  5351 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.532845  5351 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.532904  5351 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.534375  5351 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.534438  5351 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
00:28:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.534711  5372 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap starting.
00:28:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.545320  5351 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.131:41999
00:28:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.545342  5479 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.131:41999 every 8 connection(s)
00:28:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.546263  5351 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.553112  5372 log.cc:826] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Log is configured to *not* fsync() on all Append() calls
00:28:23.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.554042  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 5351
00:28:23.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:23.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.557900  5482 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:23.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.558120  5482 heartbeater.cc:461] Registering TS with master...
00:28:23.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.558676  5482 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.560446  4136 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:23.562 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:23.563 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.129:43753
00:28:23.564 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:23.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.564225  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4996
00:28:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.567389  5481 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.567477  5481 heartbeater.cc:461] Registering TS with master...
00:28:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.567684  5481 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.567737  5480 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.567831  5480 heartbeater.cc:461] Registering TS with master...
00:28:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.567869  5372 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.568013  5480 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.568248  5372 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap complete.
00:28:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.568589  4064 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.568805  4208 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.569427  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.131:45825
00:28:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.569603  5372 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent bootstrapping tablet: real 0.035s	user 0.023s	sys 0.008s
00:28:23.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.572602  5372 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:23.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.573063  5372 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Initialized, Role: FOLLOWER
00:28:23.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.573536  5372 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:23.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.574563  5372 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent starting tablet: real 0.005s	user 0.006s	sys 0.000s
00:28:23.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.574692  5481 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:23.575 [DEBUG - kudu-nio-2] (Connection.java:239) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] handling channelInactive
00:28:23.575 [DEBUG - kudu-nio-2] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state READY due to: connection closed
00:28:23.577 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:23.577 [DEBUG - Time-limited test] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:23.578 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.578 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.579 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.596 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:23.596 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:23.596 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [18ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1187031163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@2ca4f43e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60018027 -> wakeup thread Time-limited test)) after lookup
00:28:23.597 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:23.598 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.598 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.598 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.615 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:23.615 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:23.616 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [18ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [21ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [22ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1187031163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@2ca4f43e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60018027 -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427))) after lookup
00:28:23.616 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:23.617 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.617 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.617 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.636 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:23.636 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:23.636 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [18ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [21ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [22ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [40ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [40ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1187031163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@2ca4f43e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60018027 -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568))) after lookup
00:28:23.637 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:23.638 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.638 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.638 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.675 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:23.675 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:23.676 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=98), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [18ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [21ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [22ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [40ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [40ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [60ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [61ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [61ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [98ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1187031163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@2ca4f43e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60018027 -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022))) after lookup
00:28:23.676 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:23.677 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.677 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.677 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.696 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:23.696 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:23.696 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=119), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [18ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [21ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [22ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [40ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [40ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [60ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [61ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [61ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [98ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [99ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [100ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [101ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [118ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1187031163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@2ca4f43e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022) -> (continuation of Deferred@1478513537 after retry RPC@864586394), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60018027 -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022) -> (continuation of Deferred@1478513537 after retry RPC@864586394))) after lookup
00:28:23.697 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:23.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:23.698 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:24.356 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:24.356 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:24.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=779), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [18ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [21ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [22ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [40ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [40ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [60ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [61ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [61ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [98ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [99ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [100ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [101ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [118ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [120ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [121ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [121ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [776ms] refreshing cache from master, [776ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [779ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1187031163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@2ca4f43e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022) -> (continuation of Deferred@1478513537 after retry RPC@864586394) -> (continuation of Deferred@1354146181 after retry RPC@656223571), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60018027 -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022) -> (continuation of Deferred@1478513537 after retry RPC@864586394) -> (continuation of Deferred@1354146181 after retry RPC@656223571))) after lookup
00:28:24.358 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:24.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:24.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:24.359 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.876157  5493 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 6 FOLLOWER]: Starting pre-election (detected failure of leader f209795a44804bb3aea71582dc28cf0f)
00:28:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.876298  5493 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.876801  5493 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:24.878530  5194 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:24.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.881791  5434 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" is_pre_election: true
00:28:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.882054  5434 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 731068d234f34dc09763e08ded790c5d in term 5.
00:28:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.882676  5194 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, 731068d234f34dc09763e08ded790c5d; no voters: f209795a44804bb3aea71582dc28cf0f
00:28:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.882845  5493 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 6 FOLLOWER]: Leader pre-election won for term 7
00:28:24.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.882916  5493 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 6 FOLLOWER]: Starting leader election (detected failure of leader f209795a44804bb3aea71582dc28cf0f)
00:28:24.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.882949  5493 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 6 FOLLOWER]: Advancing to term 7
00:28:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.883984  5493 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.884325  5493 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 7 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.884681  5434 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.884821  5434 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 5 FOLLOWER]: Advancing to term 7
00:28:24.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:24.885351  5194 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:24.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.886593  5434 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 731068d234f34dc09763e08ded790c5d in term 7.
00:28:24.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.886931  5194 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, 731068d234f34dc09763e08ded790c5d; no voters: f209795a44804bb3aea71582dc28cf0f
00:28:24.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.887096  5493 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 7 FOLLOWER]: Leader election won for term 7
00:28:24.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.887284  5493 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 7 LEADER]: Becoming Leader. State: Replica: 731068d234f34dc09763e08ded790c5d, State: Running, Role: LEADER
00:28:24.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.887569  5493 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.890051  4064 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d reported cstate change: term changed from 6 to 7, leader changed from f209795a44804bb3aea71582dc28cf0f (127.3.237.129) to 731068d234f34dc09763e08ded790c5d (127.3.237.130). New cstate: current_term: 7 leader_uuid: "731068d234f34dc09763e08ded790c5d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: HEALTHY } } }
00:28:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.401925  5434 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 7 FOLLOWER]: Refusing update from remote peer 731068d234f34dc09763e08ded790c5d: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
00:28:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.402652  5510 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
00:28:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.407408  5512 mvcc.cc:204] Tried to move back new op lower bound from 7305095181817307136 to 7305095177631481856. Current Snapshot: MvccSnapshot[applied={T|T < 7305095181759344640}]
00:28:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.411803  5194 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d -> Peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Couldn't send request to peer f209795a44804bb3aea71582dc28cf0f. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:27.035 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:27.036 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:27.036 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=3459), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [18ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [19ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [21ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [22ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [40ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [40ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [60ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [61ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [61ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [98ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [99ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [100ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [101ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [118ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [120ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [121ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [121ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [776ms] refreshing cache from master, [776ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [779ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [780ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [782ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [782ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [3456ms] refreshing cache from master, [3456ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [3458ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1187031163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@2ca4f43e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022) -> (continuation of Deferred@1478513537 after retry RPC@864586394) -> (continuation of Deferred@1354146181 after retry RPC@656223571) -> (continuation of Deferred@467978446 after retry RPC@1432160596), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60018027 -> wakeup thread Time-limited test -> (continuation of Deferred@1457189991 after retry RPC@976635427) -> (continuation of Deferred@2144077284 after retry RPC@2009289568) -> (continuation of Deferred@1576952194 after retry RPC@818635022) -> (continuation of Deferred@1478513537 after retry RPC@864586394) -> (continuation of Deferred@1354146181 after retry RPC@656223571) -> (continuation of Deferred@467978446 after retry RPC@1432160596))) after lookup
00:28:27.037 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:27.038 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] Successfully connected to peer
00:28:27.054 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x3f17ea76, L:/127.0.0.1:35570 - R:/127.3.237.130:41841] using TOKEN/null
00:28:27.078 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.129:43753
00:28:27.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.080924  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:43753
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32913
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:27.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.189005  5535 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:27.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.189186  5535 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:27.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.189229  5535 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:27.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.192445  5535 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:27.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.192538  5535 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.129
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.196257  5535 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:43753
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32913
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5535
00:28:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.197389  5535 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.198500  5535 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:27.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.204725  5543 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.204751  5542 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.204806  5535 server_base.cc:1061] running on GCE node
00:28:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.204967  5545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.205513  5535 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.206110  5535 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:27.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.207281  5535 hybrid_clock.cc:648] HybridClock initialized: now 1783470507207262 us; error 30 us; skew 500 ppm
00:28:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.208966  5535 webserver.cc:534] Webserver started at http://127.3.237.129:32913/ using document root <none> and password file <none>
00:28:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.209542  5535 fs_manager.cc:362] Metadata directory not provided
00:28:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.209625  5535 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:27.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.212797  5535 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
00:28:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.214643  5551 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:27.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.215502  5535 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.215626  5535 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.216090  5535 fs_report.cc:389] FS layout report
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/data
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.224376  5535 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.225102  5535 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.225311  5535 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.225910  5535 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.227137  5558 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.230064  5535 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.230141  5535 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.230203  5535 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:27.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.231544  5535 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:27.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.231616  5535 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
00:28:27.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.231925  5558 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap starting.
00:28:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.242362  5535 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.129:43753
00:28:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.242450  5665 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.129:43753 every 8 connection(s)
00:28:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.243350  5535 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:27.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:27.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.247624  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 5535
00:28:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.251590  5558 log.cc:826] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Log is configured to *not* fsync() on all Append() calls
00:28:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.254637  5668 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.254884  5668 heartbeater.cc:461] Registering TS with master...
00:28:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.255465  5668 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.257350  4136 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.258170  5666 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.258249  5666 heartbeater.cc:461] Registering TS with master...
00:28:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.258451  5666 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:27.259 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:27.260 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.130:41841
00:28:27.260 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.259513  5667 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.259593  5667 heartbeater.cc:461] Registering TS with master...
00:28:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.259793  5667 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.259954  4207 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.260673  4064 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.261186  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 5176
00:28:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.261615  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.129:33965
00:28:27.268 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:27.269 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] handling channelInactive
00:28:27.269 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state READY due to: connection closed
00:28:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.269577  5558 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.269887  5558 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap complete.
00:28:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.271384  5558 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent bootstrapping tablet: real 0.040s	user 0.025s	sys 0.007s
00:28:27.273 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.274499  5558 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.274883  5558 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Initialized, Role: FOLLOWER
00:28:27.275 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.275367  5558 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:27.275 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.275 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.276350  5558 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent starting tablet: real 0.005s	user 0.007s	sys 0.000s
00:28:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.277035  5667 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:27.296 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:27.297 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:27.297 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=3, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [6ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [6ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1523774395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@4697b5b6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@259e76d6 -> wakeup thread Time-limited test)) after lookup
00:28:27.297 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:27.298 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.298 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.299 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.316 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:27.316 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:27.316 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=5, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [6ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [6ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [30ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [30ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [46ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1523774395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@4697b5b6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@259e76d6 -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079))) after lookup
00:28:27.317 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:27.317 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.318 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.318 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.356 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:27.357 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:27.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=7, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [6ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [6ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [30ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [30ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [46ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [47ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [49ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [49ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [87ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1523774395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@4697b5b6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@259e76d6 -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480))) after lookup
00:28:27.358 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:27.359 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.359 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.359 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.596 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:27.597 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:27.597 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=9, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=328), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [6ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [6ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [30ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [30ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [46ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [47ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [49ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [49ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [87ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [89ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [90ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [90ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [324ms] refreshing cache from master, [324ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [327ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1523774395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@4697b5b6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480) -> (continuation of Deferred@240848103 after retry RPC@1348829835), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@259e76d6 -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480) -> (continuation of Deferred@240848103 after retry RPC@1348829835))) after lookup
00:28:27.598 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:27.599 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.599 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:27.599 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:28.576 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:28.576 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:28.577 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=11, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1308), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [6ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [6ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [30ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [30ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [46ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [47ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [49ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [49ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [87ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [89ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [90ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [90ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [324ms] refreshing cache from master, [324ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [327ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [329ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [330ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [330ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1304ms] refreshing cache from master, [1305ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [1307ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1523774395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@4697b5b6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480) -> (continuation of Deferred@240848103 after retry RPC@1348829835) -> (continuation of Deferred@364152675 after retry RPC@1773743748), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@259e76d6 -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480) -> (continuation of Deferred@240848103 after retry RPC@1348829835) -> (continuation of Deferred@364152675 after retry RPC@1773743748))) after lookup
00:28:28.577 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.578455  5679 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 731068d234f34dc09763e08ded790c5d)
00:28:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.578608  5679 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:28.579 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:28.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:28.579 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.579942  5679 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:28.580709  5368 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:28.581773  5368 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:28.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.585193  5620 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f" is_pre_election: true
00:28:28.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.585449  5620 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 6.
00:28:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.586133  5369 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 731068d234f34dc09763e08ded790c5d
00:28:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.586356  5679 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 7 FOLLOWER]: Leader pre-election won for term 8
00:28:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.586433  5679 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 731068d234f34dc09763e08ded790c5d)
00:28:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.586478  5679 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 7 FOLLOWER]: Advancing to term 8
00:28:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.587307  5679 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.587692  5679 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 8 election: Requested vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.588106  5620 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.588246  5620 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 6 FOLLOWER]: Advancing to term 8
00:28:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:28.588794  5368 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.589946  5620 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 8.
00:28:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.590322  5369 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 731068d234f34dc09763e08ded790c5d
00:28:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.590507  5679 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 8 FOLLOWER]: Leader election won for term 8
00:28:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.590755  5679 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 8 LEADER]: Becoming Leader. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Running, Role: LEADER
00:28:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.591058  5679 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.593315  4064 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 reported cstate change: term changed from 7 to 8, leader changed from 731068d234f34dc09763e08ded790c5d (127.3.237.130) to 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131). New cstate: current_term: 8 leader_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:28.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.994993  5620 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 8 FOLLOWER]: Refusing update from remote peer 2ddcbfc125f04fd592241ffd11b0dfb7: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
00:28:28.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.995790  5679 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
00:28:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.004539  5368 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 -> Peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Couldn't send request to peer 731068d234f34dc09763e08ded790c5d. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:29.456 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:29.456 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:29.457 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=13, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2187), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [6ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [6ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [30ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [30ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [46ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [47ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [49ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [49ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [87ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [89ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [90ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [90ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [324ms] refreshing cache from master, [324ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [327ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [329ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [330ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [330ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1304ms] refreshing cache from master, [1305ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [1307ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [1308ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [1310ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [1310ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2184ms] refreshing cache from master, [2184ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2187ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1523774395(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@4697b5b6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480) -> (continuation of Deferred@240848103 after retry RPC@1348829835) -> (continuation of Deferred@364152675 after retry RPC@1773743748) -> (continuation of Deferred@1074992902 after retry RPC@1618926734), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@259e76d6 -> wakeup thread Time-limited test -> (continuation of Deferred@1862089923 after retry RPC@197240079) -> (continuation of Deferred@762420900 after retry RPC@723435480) -> (continuation of Deferred@240848103 after retry RPC@1348829835) -> (continuation of Deferred@364152675 after retry RPC@1773743748) -> (continuation of Deferred@1074992902 after retry RPC@1618926734))) after lookup
00:28:29.457 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:29.459 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] Successfully connected to peer
00:28:29.475 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x954308bd, L:/127.0.0.1:36062 - R:/127.3.237.131:41999] using TOKEN/null
00:28:29.491 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.130:41841
00:28:29.492 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.492632  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:41841
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46813
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:29.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.596525  5708 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:29.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.596698  5708 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:29.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.596740  5708 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:29.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.600126  5708 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:29.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.600222  5708 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.130
00:28:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.603768  5708 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:41841
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46813
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5708
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.604750  5708 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.605729  5708 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.611878  5713 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.611884  5714 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.612150  5708 server_base.cc:1061] running on GCE node
00:28:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.611886  5716 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.612722  5708 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.613269  5708 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:29.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.614421  5708 hybrid_clock.cc:648] HybridClock initialized: now 1783470509614400 us; error 33 us; skew 500 ppm
00:28:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.616107  5708 webserver.cc:534] Webserver started at http://127.3.237.130:46813/ using document root <none> and password file <none>
00:28:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.616644  5708 fs_manager.cc:362] Metadata directory not provided
00:28:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.616729  5708 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.619885  5708 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
00:28:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.621614  5722 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.622443  5708 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.622577  5708 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731068d234f34dc09763e08ded790c5d"
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.622952  5708 fs_report.cc:389] FS layout report
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/data
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.652084  5708 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.652628  5708 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.652762  5708 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.653245  5708 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.654346  5729 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:29.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.657362  5708 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:29.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.657415  5708 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:29.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.657461  5708 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.658798  5708 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:29.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.658840  5708 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
00:28:29.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.659191  5729 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap starting.
00:28:29.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.670452  5708 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.130:41841
00:28:29.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.670517  5836 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.130:41841 every 8 connection(s)
00:28:29.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.671480  5708 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:29.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.677484  5729 log.cc:826] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Log is configured to *not* fsync() on all Append() calls
00:28:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.678238  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 5708
00:28:29.680 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:29.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.685885  5838 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:29.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.685920  5839 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:29.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.686115  5838 heartbeater.cc:461] Registering TS with master...
00:28:29.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.686117  5839 heartbeater.cc:461] Registering TS with master...
00:28:29.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.686683  5838 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:29.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.686683  5839 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:29.688 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:29.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.688683  4064 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:29.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.688879  4135 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:29.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.689606  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.130:59057
00:28:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.689932  5837 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.690016  5837 heartbeater.cc:461] Registering TS with master...
00:28:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.690213  5837 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:29.690 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.131:41999
00:28:29.690 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:29.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.691069  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 5351
00:28:29.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.691089  4208 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:29.699 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] handling channelInactive
00:28:29.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state READY due to: connection closed
00:28:29.700 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:29.701 [DEBUG - Time-limited test] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:29.702 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.702 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.702 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.703578  5729 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:29.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.704028  5729 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap complete.
00:28:29.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.705083  5729 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent bootstrapping tablet: real 0.046s	user 0.022s	sys 0.015s
00:28:29.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.707033  5729 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:29.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.707336  5729 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 731068d234f34dc09763e08ded790c5d, State: Initialized, Role: FOLLOWER
00:28:29.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.707715  5729 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:29.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.708390  5838 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:29.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.708591  5729 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent starting tablet: real 0.003s	user 0.004s	sys 0.000s
00:28:29.716 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:29.716 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:29.716 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [16ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1237857887(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@693364af -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@d70cfb3 -> wakeup thread Time-limited test)) after lookup
00:28:29.717 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:29.717 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.718 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.718 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.735 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:29.735 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:29.736 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [16ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [16ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1237857887(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@693364af -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@d70cfb3 -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955))) after lookup
00:28:29.736 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:29.737 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.737 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.737 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.755 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:29.756 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:29.756 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [16ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [16ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [55ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1237857887(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@693364af -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@d70cfb3 -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673))) after lookup
00:28:29.756 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:29.757 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.758 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.758 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.816 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:29.816 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:29.817 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=116), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [16ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [16ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [55ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [58ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [58ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [116ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1237857887(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@693364af -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@d70cfb3 -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997))) after lookup
00:28:29.818 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:29.819 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.819 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.819 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.996 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:29.997 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:29.997 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=296), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [16ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [16ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [55ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [58ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [58ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [116ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [117ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [119ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [119ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [293ms] refreshing cache from master, [294ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [296ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1237857887(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@693364af -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997) -> (continuation of Deferred@144769630 after retry RPC@1926872927), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@d70cfb3 -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997) -> (continuation of Deferred@144769630 after retry RPC@1926872927))) after lookup
00:28:29.998 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:29.999 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.999 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:29.999 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:30.716 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:30.716 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:30.717 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1016), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [16ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [16ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [55ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [58ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [58ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [116ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [117ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [119ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [119ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [293ms] refreshing cache from master, [294ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [296ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [298ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [299ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [299ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [1013ms] refreshing cache from master, [1013ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [1016ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1237857887(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@693364af -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997) -> (continuation of Deferred@144769630 after retry RPC@1926872927) -> (continuation of Deferred@1738443887 after retry RPC@1476588676), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@d70cfb3 -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997) -> (continuation of Deferred@144769630 after retry RPC@1926872927) -> (continuation of Deferred@1738443887 after retry RPC@1476588676))) after lookup
00:28:30.718 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:30.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:30.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:30.719 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:30.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.991516  5850 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:30.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.991660  5850 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:30.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.992743  5850 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.993417  5555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:30.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.994424  5555 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.997717  5791 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d" is_pre_election: true
00:28:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.998023  5791 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 7.
00:28:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.998730  5554 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.998986  5850 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 8 FOLLOWER]: Leader pre-election won for term 9
00:28:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.999053  5850 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 8 FOLLOWER]: Starting leader election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.999083  5850 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 8 FOLLOWER]: Advancing to term 9
00:28:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.999907  5850 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.000298  5850 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 9 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.000703  5791 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d"
00:28:31.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.000888  5791 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 7 FOLLOWER]: Advancing to term 9
00:28:31.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:31.001359  5555 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:31.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.002619  5791 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 9.
00:28:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.002938  5554 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.003094  5850 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 9 FOLLOWER]: Leader election won for term 9
00:28:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.003289  5850 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 9 LEADER]: Becoming Leader. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Running, Role: LEADER
00:28:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.003561  5850 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.005980  4064 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f reported cstate change: term changed from 8 to 9, leader changed from 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131) to f209795a44804bb3aea71582dc28cf0f (127.3.237.129). New cstate: current_term: 9 leader_uuid: "f209795a44804bb3aea71582dc28cf0f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:31.501379  5555 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f -> Peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Couldn't send request to peer 2ddcbfc125f04fd592241ffd11b0dfb7. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.580689  5791 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 9 FOLLOWER]: Refusing update from remote peer f209795a44804bb3aea71582dc28cf0f: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
00:28:31.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:31.581368  5850 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
00:28:33.736 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:33.737 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:33.737 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=4036), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [2ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [16ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [16ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [35ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [36ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [37ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [37ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [55ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [56ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [58ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [58ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [116ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [117ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [119ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [119ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [293ms] refreshing cache from master, [294ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [296ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [298ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [299ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [299ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [1013ms] refreshing cache from master, [1013ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [1016ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [1017ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [1019ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [1019ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [4034ms] refreshing cache from master, [4034ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [4036ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1237857887(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@693364af -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997) -> (continuation of Deferred@144769630 after retry RPC@1926872927) -> (continuation of Deferred@1738443887 after retry RPC@1476588676) -> (continuation of Deferred@744149609 after retry RPC@1672721842), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@d70cfb3 -> wakeup thread Time-limited test -> (continuation of Deferred@1691948092 after retry RPC@1868517955) -> (continuation of Deferred@1583248267 after retry RPC@1258745673) -> (continuation of Deferred@338568358 after retry RPC@1747905997) -> (continuation of Deferred@144769630 after retry RPC@1926872927) -> (continuation of Deferred@1738443887 after retry RPC@1476588676) -> (continuation of Deferred@744149609 after retry RPC@1672721842))) after lookup
00:28:33.738 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:33.739 [DEBUG - kudu-nio-1] (Connection.java:757) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] Successfully connected to peer
00:28:33.757 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x64281ab6, L:/127.0.0.1:42778 - R:/127.3.237.129:43753] using TOKEN/null
00:28:33.773 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.131:41999
00:28:33.773 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.774299  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:33.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.875134  5555 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f -> Peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Couldn't send request to peer 2ddcbfc125f04fd592241ffd11b0dfb7. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.878103  5888 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.878271  5888 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.878316  5888 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:33.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.881491  5888 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:33.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.881587  5888 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.131
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.885172  5888 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5888
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.886071  5888 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.887362  5888 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.894451  5898 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.894497  5896 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.894469  5895 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.894847  5888 server_base.cc:1061] running on GCE node
00:28:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.895226  5888 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.895735  5888 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:33.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.896893  5888 hybrid_clock.cc:648] HybridClock initialized: now 1783470513896872 us; error 38 us; skew 500 ppm
00:28:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.898610  5888 webserver.cc:534] Webserver started at http://127.3.237.131:41753/ using document root <none> and password file <none>
00:28:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.899092  5888 fs_manager.cc:362] Metadata directory not provided
00:28:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.899158  5888 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:33.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.902040  5888 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.001s	sys 0.000s
00:28:33.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.903635  5904 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:33.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.904633  5888 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.904721  5888 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.905040  5888 fs_report.cc:389] FS layout report
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/data
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.923031  5888 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.923552  5888 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.923723  5888 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.924289  5888 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.925567  5911 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.928623  5888 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.928675  5888 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.928717  5888 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.929950  5888 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.929989  5888 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
00:28:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.930321  5911 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap starting.
00:28:33.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.940724  5888 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.131:41999
00:28:33.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.940807  6018 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.131:41999 every 8 connection(s)
00:28:33.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.941730  5888 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.948573  5911 log.cc:826] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Log is configured to *not* fsync() on all Append() calls
00:28:33.950 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.950142  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 5888
00:28:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.953287  6021 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.953486  6019 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.953536  6021 heartbeater.cc:461] Registering TS with master...
00:28:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.953567  6019 heartbeater.cc:461] Registering TS with master...
00:28:33.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.953558  6020 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:33.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.954120  6021 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:33.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.954187  6020 heartbeater.cc:461] Registering TS with master...
00:28:33.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.954337  6020 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:33.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.954118  6019 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:33.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.955942  4136 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:33.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.956182  4208 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:33.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.956743  4063 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.957698  4063 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.131:34329
00:28:33.959 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:33.960 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.129:43753
00:28:33.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.960927  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 5535
00:28:33.968 [DEBUG - kudu-nio-1] (Connection.java:239) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] handling channelInactive
00:28:33.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state READY due to: connection closed
00:28:33.969 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:33.969 [DEBUG - Time-limited test] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:33.970 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.970 [DEBUG - kudu-nio-2] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.971 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.974077  5911 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:33.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.974470  5911 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap complete.
00:28:33.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.975760  5911 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent bootstrapping tablet: real 0.046s	user 0.021s	sys 0.014s
00:28:33.975 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:33.976 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:33.976 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1601901286(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@74d5d043 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fb790c7 -> wakeup thread Time-limited test)) after lookup
00:28:33.977 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:33.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.978381  5911 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:33.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.978726  5911 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Initialized, Role: FOLLOWER
00:28:33.978 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.979 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.979 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.979233  5911 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:33.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.980264  5911 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent starting tablet: real 0.004s	user 0.003s	sys 0.001s
00:28:33.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.980872  6020 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:33.996 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:33.996 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:33.997 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [7ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [10ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [10ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1601901286(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@74d5d043 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fb790c7 -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103))) after lookup
00:28:33.997 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:33.998 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.998 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:33.998 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.036 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:34.036 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:34.037 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [7ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [10ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [10ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1601901286(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@74d5d043 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fb790c7 -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149))) after lookup
00:28:34.037 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:34.038 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.039 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.039 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.096 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:34.096 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:34.097 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=127), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [7ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [10ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [10ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [68ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [70ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [70ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [127ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1601901286(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@74d5d043 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149) -> (continuation of Deferred@1648285675 after retry RPC@2119375322), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fb790c7 -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149) -> (continuation of Deferred@1648285675 after retry RPC@2119375322))) after lookup
00:28:34.097 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:34.099 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.099 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.099 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.376 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:34.376 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:34.377 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=407), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [7ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [10ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [10ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [68ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [70ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [70ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [127ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [128ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [130ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [130ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [404ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [407ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1601901286(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@74d5d043 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149) -> (continuation of Deferred@1648285675 after retry RPC@2119375322) -> (continuation of Deferred@2044394965 after retry RPC@530691837), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fb790c7 -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149) -> (continuation of Deferred@1648285675 after retry RPC@2119375322) -> (continuation of Deferred@2044394965 after retry RPC@530691837))) after lookup
00:28:34.377 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:34.378 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.379 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:34.379 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753
00:28:35.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.273572  6032 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 9 FOLLOWER]: Starting pre-election (detected failure of leader f209795a44804bb3aea71582dc28cf0f)
00:28:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.273749  6032 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.275094  6032 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.276724  5726 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.277531  5726 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:35.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.280797  5970 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" is_pre_election: true
00:28:35.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.281041  5970 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 731068d234f34dc09763e08ded790c5d in term 8.
00:28:35.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.281703  5726 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, 731068d234f34dc09763e08ded790c5d; no voters: f209795a44804bb3aea71582dc28cf0f
00:28:35.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.281980  6032 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 9 FOLLOWER]: Leader pre-election won for term 10
00:28:35.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.282037  6032 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 9 FOLLOWER]: Starting leader election (detected failure of leader f209795a44804bb3aea71582dc28cf0f)
00:28:35.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.282064  6032 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 9 FOLLOWER]: Advancing to term 10
00:28:35.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.282796  6032 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:35.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.283123  6032 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 10 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:35.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.283644  5970 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "731068d234f34dc09763e08ded790c5d" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:35.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.283788  5970 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 8 FOLLOWER]: Advancing to term 10
00:28:35.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.284408  5726 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111)
00:28:35.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.285454  5970 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 731068d234f34dc09763e08ded790c5d in term 10.
00:28:35.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.285780  5726 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, 731068d234f34dc09763e08ded790c5d; no voters: f209795a44804bb3aea71582dc28cf0f
00:28:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.285928  6032 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 10 FOLLOWER]: Leader election won for term 10
00:28:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.286098  6032 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 10 LEADER]: Becoming Leader. State: Replica: 731068d234f34dc09763e08ded790c5d, State: Running, Role: LEADER
00:28:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.286316  6032 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:35.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.288858  4063 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d reported cstate change: term changed from 9 to 10, leader changed from f209795a44804bb3aea71582dc28cf0f (127.3.237.129) to 731068d234f34dc09763e08ded790c5d (127.3.237.130). New cstate: current_term: 10 leader_uuid: "731068d234f34dc09763e08ded790c5d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: HEALTHY } } }
00:28:35.536 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:35.536 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:35.537 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1567), Traces: [0ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [2ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [2ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [6ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [7ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [10ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [10ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [24ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [27ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [28ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [29ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [29ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [67ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [68ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [70ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [70ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [124ms] refreshing cache from master, [124ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [127ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [128ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [130ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [130ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [404ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [407ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [408ms] sending RPC to server f209795a44804bb3aea71582dc28cf0f, [410ms] received response from server f209795a44804bb3aea71582dc28cf0f: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [410ms] delaying RPC due to: Network error: Failed to connect to peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753): Connection refused: /127.3.237.129:43753, [1564ms] refreshing cache from master, [1564ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [1567ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1601901286(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@74d5d043 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149) -> (continuation of Deferred@1648285675 after retry RPC@2119375322) -> (continuation of Deferred@2044394965 after retry RPC@530691837) -> (continuation of Deferred@1679130777 after retry RPC@1980880056), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fb790c7 -> wakeup thread Time-limited test -> (continuation of Deferred@209132991 after retry RPC@174843103) -> (continuation of Deferred@355719216 after retry RPC@1084984149) -> (continuation of Deferred@1648285675 after retry RPC@2119375322) -> (continuation of Deferred@2044394965 after retry RPC@530691837) -> (continuation of Deferred@1679130777 after retry RPC@1980880056))) after lookup
00:28:35.537 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:35.538 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] Successfully connected to peer
00:28:35.552 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfc0fc2f0, L:/127.0.0.1:42406 - R:/127.3.237.130:41841] using TOKEN/null
00:28:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.555900  5970 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 10 FOLLOWER]: Refusing update from remote peer 731068d234f34dc09763e08ded790c5d: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 34. (index mismatch)
00:28:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.556227  5726 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d -> Peer f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753): Couldn't send request to peer f209795a44804bb3aea71582dc28cf0f. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.129:43753: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.556669  6032 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
00:28:35.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.563060  6051 mvcc.cc:204] Tried to move back new op lower bound from 7305095231710281728 to 7305095230615007232. Current Snapshot: MvccSnapshot[applied={T|T < 7305095231710281728}]
00:28:35.578 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.129:43753
00:28:35.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.578900  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:43753
00:28:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32913
00:28:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.677702  6060 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.677919  6060 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.677981  6060 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.681260  6060 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.681403  6060 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.129
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.685132  6060 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.129:43753
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.129
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32913
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6060
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.129
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/logs
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.686378  6060 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.687436  6060 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.693362  6065 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.693374  6068 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:35.693431  6066 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.693801  6060 server_base.cc:1061] running on GCE node
00:28:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.694103  6060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.694577  6060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.695763  6060 hybrid_clock.cc:648] HybridClock initialized: now 1783470515695738 us; error 33 us; skew 500 ppm
00:28:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.700472  6060 webserver.cc:534] Webserver started at http://127.3.237.129:32913/ using document root <none> and password file <none>
00:28:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.700976  6060 fs_manager.cc:362] Metadata directory not provided
00:28:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.701026  6060 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:35.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.704186  6060 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.001s	sys 0.002s
00:28:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.705982  6074 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.706887  6060 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.706977  6060 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:05 on dist-test-slave-q60k"
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.707312  6060 fs_report.cc:389] FS layout report
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/wal
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/data
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.722573  6060 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.723117  6060 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.723248  6060 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.723706  6060 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.725059  6081 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.728188  6060 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.728241  6060 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.728286  6060 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.729612  6060 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.729655  6060 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
00:28:35.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.729991  6081 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap starting.
00:28:35.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.740428  6060 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.129:43753
00:28:35.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.740509  6188 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.129:43753 every 8 connection(s)
00:28:35.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.741384  6060 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-0/data/info.pb
00:28:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.744405  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 6060
00:28:35.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.747432  6081 log.cc:826] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Log is configured to *not* fsync() on all Append() calls
00:28:35.754 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:35.755 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.130:41841
00:28:35.755 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.755551  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 5708
00:28:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.755650  6191 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.755857  6191 heartbeater.cc:461] Registering TS with master...
00:28:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.756377  6191 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.758061  6190 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.758142  6190 heartbeater.cc:461] Registering TS with master...
00:28:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.758337  6190 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.758755  6189 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.758823  6189 heartbeater.cc:461] Registering TS with master...
00:28:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.758977  6189 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.759625  4063 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.760010  4136 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.760557  4063 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.129:47169
00:28:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.761346  4208 ts_manager.cc:194] Re-registered known tserver with Master: f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753)
00:28:35.764 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] handling channelInactive
00:28:35.764 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state READY due to: connection closed
00:28:35.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:35.765 [DEBUG - Time-limited test] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:35.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.767 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.773626  6081 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.774770  6081 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Bootstrap complete.
00:28:35.777 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:35.777 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:35.778 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [2ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [12ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1154413356(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@3c025b19 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f672372 -> wakeup thread Time-limited test)) after lookup
00:28:35.778 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.776160  6081 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent bootstrapping tablet: real 0.046s	user 0.027s	sys 0.011s
00:28:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.781360  6081 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.781813  6081 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Initialized, Role: FOLLOWER
00:28:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.782299  6081 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.783190  6190 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:35.786 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:35.783286  6081 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f: Time spent starting tablet: real 0.007s	user 0.004s	sys 0.001s
00:28:35.786 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.786 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.796 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:35.796 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:35.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [2ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [12ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [13ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [21ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [21ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1154413356(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@3c025b19 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f672372 -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169))) after lookup
00:28:35.797 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:35.800 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.800 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.800 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.836 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:35.836 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:35.837 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [2ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [12ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [13ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [21ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [21ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [32ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [35ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [35ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [71ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1154413356(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@3c025b19 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f672372 -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876))) after lookup
00:28:35.837 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:35.838 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.838 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.838 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.875 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:35.875 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:35.876 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=110), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [2ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [12ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [13ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [21ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [21ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [32ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [35ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [35ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [71ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [72ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [73ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [73ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [110ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1154413356(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@3c025b19 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876) -> (continuation of Deferred@278137871 after retry RPC@1942615591), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f672372 -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876) -> (continuation of Deferred@278137871 after retry RPC@1942615591))) after lookup
00:28:35.876 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:35.877 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.877 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.877 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.916 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:35.916 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)[L],f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:35.917 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=151), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [2ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [12ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [13ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [21ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [21ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [32ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [35ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [35ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [71ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [72ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [73ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [73ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [110ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [111ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [112ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [112ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [151ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1154413356(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@3c025b19 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876) -> (continuation of Deferred@278137871 after retry RPC@1942615591) -> (continuation of Deferred@604343312 after retry RPC@406929017), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f672372 -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876) -> (continuation of Deferred@278137871 after retry RPC@1942615591) -> (continuation of Deferred@604343312 after retry RPC@406929017))) after lookup
00:28:35.917 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] connecting to peer
00:28:35.918 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.918 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] cleaning up while in state CONNECTING due to: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:35.918 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841
00:28:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.077847  6202 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 731068d234f34dc09763e08ded790c5d)
00:28:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.077988  6202 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.078976  6202 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.079733  5907 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.080742  5907 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.084178  6143 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f" is_pre_election: true
00:28:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.084472  6143 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 9.
00:28:37.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.085117  5908 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 731068d234f34dc09763e08ded790c5d
00:28:37.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.085350  6202 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 10 FOLLOWER]: Leader pre-election won for term 11
00:28:37.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.085412  6202 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 731068d234f34dc09763e08ded790c5d)
00:28:37.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.085451  6202 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 10 FOLLOWER]: Advancing to term 11
00:28:37.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.086232  6202 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:37.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.086580  6202 leader_election.cc:290] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 11 election: Requested vote from peers f209795a44804bb3aea71582dc28cf0f (127.3.237.129:43753), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.086939  6143 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "f209795a44804bb3aea71582dc28cf0f"
00:28:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.087083  6143 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 9 FOLLOWER]: Advancing to term 11
00:28:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.087661  5907 leader_election.cc:336] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111)
00:28:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.088866  6143 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ddcbfc125f04fd592241ffd11b0dfb7 in term 11.
00:28:37.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.089169  5908 leader_election.cc:304] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2ddcbfc125f04fd592241ffd11b0dfb7, f209795a44804bb3aea71582dc28cf0f; no voters: 731068d234f34dc09763e08ded790c5d
00:28:37.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.089371  6202 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 11 FOLLOWER]: Leader election won for term 11
00:28:37.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.089566  6202 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 11 LEADER]: Becoming Leader. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Running, Role: LEADER
00:28:37.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.089847  6202 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.092283  4063 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 reported cstate change: term changed from 10 to 11, leader changed from 731068d234f34dc09763e08ded790c5d (127.3.237.130) to 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131). New cstate: current_term: 11 leader_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.599889  6143 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 11 FOLLOWER]: Refusing update from remote peer 2ddcbfc125f04fd592241ffd11b0dfb7: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 37. (index mismatch)
00:28:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.600594  6219 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
00:28:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.605278  6221 mvcc.cc:204] Tried to move back new op lower bound from 7305095231794003968 to 7305095230615007232. Current Snapshot: MvccSnapshot[applied={T|T < 7305095231710281728}]
00:28:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.609338  5907 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 -> Peer 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841): Couldn't send request to peer 731068d234f34dc09763e08ded790c5d. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.130:41841: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:37.876 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:37.876 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:37.876 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2111), Traces: [0ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [2ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [12ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [13ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [21ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [21ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [31ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [32ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [35ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [35ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [71ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [72ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [73ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [73ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [110ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [111ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [112ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [112ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [151ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [152ms] sending RPC to server 731068d234f34dc09763e08ded790c5d, [153ms] received response from server 731068d234f34dc09763e08ded790c5d: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [153ms] delaying RPC due to: Network error: Failed to connect to peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841): Connection refused: /127.3.237.130:41841, [2108ms] refreshing cache from master, [2108ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2110ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@1154413356(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@3c025b19 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876) -> (continuation of Deferred@278137871 after retry RPC@1942615591) -> (continuation of Deferred@604343312 after retry RPC@406929017) -> (continuation of Deferred@27841242 after retry RPC@269190752), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f672372 -> wakeup thread Time-limited test -> (continuation of Deferred@179064700 after retry RPC@2776169) -> (continuation of Deferred@2014504067 after retry RPC@705242876) -> (continuation of Deferred@278137871 after retry RPC@1942615591) -> (continuation of Deferred@604343312 after retry RPC@406929017) -> (continuation of Deferred@27841242 after retry RPC@269190752))) after lookup
00:28:37.877 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:37.878 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] Successfully connected to peer
00:28:37.900 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xddd4c312, L:/127.0.0.1:53682 - R:/127.3.237.131:41999] using TOKEN/null
00:28:37.915 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.130:41841
00:28:37.915 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.916453  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:41841
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46813
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.014173  6235 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.014343  6235 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.014385  6235 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.017470  6235 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.017566  6235 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.130
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.021066  6235 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.130:41841
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.130
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46813
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6235
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.130
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/logs
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.021966  6235 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.022902  6235 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.028746  6241 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.028753  6240 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.028807  6244 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.029366  6235 server_base.cc:1061] running on GCE node
00:28:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.029736  6235 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:38.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.030181  6235 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:38.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.031348  6235 hybrid_clock.cc:648] HybridClock initialized: now 1783470518031333 us; error 34 us; skew 500 ppm
00:28:38.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.032872  6235 webserver.cc:534] Webserver started at http://127.3.237.130:46813/ using document root <none> and password file <none>
00:28:38.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.033316  6235 fs_manager.cc:362] Metadata directory not provided
00:28:38.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.033363  6235 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:38.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.036316  6235 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.000s	sys 0.003s
00:28:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.037981  6250 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.038865  6235 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.038949  6235 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731068d234f34dc09763e08ded790c5d"
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.039279  6235 fs_report.cc:389] FS layout report
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/wal
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/data
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:38.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.071739  6235 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:38.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.072376  6235 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:38.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.072513  6235 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:38.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.072958  6235 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.074074  6257 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.077171  6235 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.077224  6235 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
00:28:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.077268  6235 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:38.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.078534  6235 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:38.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.078574  6235 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
00:28:38.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.078909  6257 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap starting.
00:28:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.089404  6235 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.130:41841
00:28:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.089488  6365 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.130:41841 every 8 connection(s)
00:28:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.090409  6235 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-1/data/info.pb
00:28:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.092494  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 6235
00:28:38.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:38.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.095758  6257 log.cc:826] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Log is configured to *not* fsync() on all Append() calls
00:28:38.101 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:38.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.106307  6367 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:38.106 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.3.237.131:41999
00:28:38.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.106510  6368 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:38.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.106565  6367 heartbeater.cc:461] Registering TS with master...
00:28:38.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.106600  6368 heartbeater.cc:461] Registering TS with master...
00:28:38.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:38.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.107160  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 5888
00:28:38.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.107146  6367 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:38.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.107151  6368 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:38.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.109294  4064 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:38.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.109498  6366 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:38.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.109587  6366 heartbeater.cc:461] Registering TS with master...
00:28:38.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.109766  6366 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.110270  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.130:35135
00:28:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.110617  4208 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.110811  4136 ts_manager.cc:194] Re-registered known tserver with Master: 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:38.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:38.116 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] handling channelInactive
00:28:38.117 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state READY due to: connection closed
00:28:38.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.126516  6257 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:38.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.126849  6257 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Bootstrap complete.
00:28:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.128010  6257 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent bootstrapping tablet: real 0.049s	user 0.031s	sys 0.011s
00:28:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.130028  6257 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.130331  6257 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 731068d234f34dc09763e08ded790c5d, State: Initialized, Role: FOLLOWER
00:28:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.130743  6257 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:38.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.131330  6367 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:38.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.131557  6257 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d: Time spent starting tablet: real 0.003s	user 0.002s	sys 0.001s
00:28:38.134 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:38.135 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.135 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.155 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:38.155 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:38.156 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=3, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [0ms] delaying RPC due to: Illegal state: connection is terminated, [17ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@511931925(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@5277db23 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b15c46c -> wakeup thread Time-limited test)) after lookup
00:28:38.156 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:38.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.157 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.157 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.176 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:38.176 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:38.176 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=5, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [0ms] delaying RPC due to: Illegal state: connection is terminated, [17ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [40ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [40ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@511931925(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@5277db23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b15c46c -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379))) after lookup
00:28:38.176 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:38.177 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.177 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.177 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.235 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:38.236 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:38.236 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=7, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=119), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [0ms] delaying RPC due to: Illegal state: connection is terminated, [17ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [40ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [40ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [59ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [60ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [60ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [118ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@511931925(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@5277db23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b15c46c -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819))) after lookup
00:28:38.236 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:38.237 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.237 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.237 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.296 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:38.296 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:38.297 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=9, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=180), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [0ms] delaying RPC due to: Illegal state: connection is terminated, [17ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [40ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [40ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [59ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [60ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [60ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [118ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [119ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [120ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [120ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [177ms] refreshing cache from master, [177ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [179ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@511931925(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@5277db23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819) -> (continuation of Deferred@169246997 after retry RPC@940198371), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b15c46c -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819) -> (continuation of Deferred@169246997 after retry RPC@940198371))) after lookup
00:28:38.297 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:38.298 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.298 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.298 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.476 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:38.476 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)[L],731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)]}]
00:28:38.477 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=11, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=360), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [0ms] delaying RPC due to: Illegal state: connection is terminated, [17ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [40ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [40ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [59ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [60ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [60ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [118ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [119ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [120ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [120ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [177ms] refreshing cache from master, [177ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [179ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [180ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [181ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [181ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [356ms] refreshing cache from master, [356ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [359ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@511931925(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@5277db23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819) -> (continuation of Deferred@169246997 after retry RPC@940198371) -> (continuation of Deferred@183343277 after retry RPC@1614974274), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b15c46c -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819) -> (continuation of Deferred@169246997 after retry RPC@940198371) -> (continuation of Deferred@183343277 after retry RPC@1614974274))) after lookup
00:28:38.477 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] connecting to peer
00:28:38.478 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.478 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:38.478 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999) for tablet c30f074164564084aed8122794b78746: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999
00:28:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.416280  6380 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.416442  6380 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.417624  6380 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:39.418273  6078 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:39.419123  6078 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.422629  6320 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d" is_pre_election: true
00:28:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.422948  6320 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 10.
00:28:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.423718  6077 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.424026  6380 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 11 FOLLOWER]: Leader pre-election won for term 12
00:28:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.424110  6380 raft_consensus.cc:493] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 11 FOLLOWER]: Starting leader election (detected failure of leader 2ddcbfc125f04fd592241ffd11b0dfb7)
00:28:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.424153  6380 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 11 FOLLOWER]: Advancing to term 12
00:28:39.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.425043  6380 raft_consensus.cc:515] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:39.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.425577  6380 leader_election.cc:290] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 12 election: Requested vote from peers 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999), 731068d234f34dc09763e08ded790c5d (127.3.237.130:41841)
00:28:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.425913  6320 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c30f074164564084aed8122794b78746" candidate_uuid: "f209795a44804bb3aea71582dc28cf0f" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "731068d234f34dc09763e08ded790c5d"
00:28:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.426050  6320 raft_consensus.cc:3060] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 10 FOLLOWER]: Advancing to term 12
00:28:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:39.426564  6078 leader_election.cc:336] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111)
00:28:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.428007  6320 raft_consensus.cc:2468] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f209795a44804bb3aea71582dc28cf0f in term 12.
00:28:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.428364  6077 leader_election.cc:304] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 731068d234f34dc09763e08ded790c5d, f209795a44804bb3aea71582dc28cf0f; no voters: 2ddcbfc125f04fd592241ffd11b0dfb7
00:28:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.428517  6380 raft_consensus.cc:2804] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 12 FOLLOWER]: Leader election won for term 12
00:28:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.428714  6380 raft_consensus.cc:697] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [term 12 LEADER]: Becoming Leader. State: Replica: f209795a44804bb3aea71582dc28cf0f, State: Running, Role: LEADER
00:28:39.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.428987  6380 consensus_queue.cc:237] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:39.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.431303  4064 catalog_manager.cc:5696] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f reported cstate change: term changed from 11 to 12, leader changed from 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131) to f209795a44804bb3aea71582dc28cf0f (127.3.237.129). New cstate: current_term: 12 leader_uuid: "f209795a44804bb3aea71582dc28cf0f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } health_report { overall_health: UNKNOWN } } }
00:28:39.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.967612  6320 raft_consensus.cc:1275] T c30f074164564084aed8122794b78746 P 731068d234f34dc09763e08ded790c5d [term 12 FOLLOWER]: Refusing update from remote peer f209795a44804bb3aea71582dc28cf0f: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 41. (index mismatch)
00:28:39.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:39.968312  6078 consensus_peers.cc:597] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f -> Peer 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999): Couldn't send request to peer 2ddcbfc125f04fd592241ffd11b0dfb7. Status: Network error: Client connection negotiation failed: client connection to 127.3.237.131:41999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:39.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.968465  6398 consensus_queue.cc:1048] T c30f074164564084aed8122794b78746 P f209795a44804bb3aea71582dc28cf0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
00:28:40.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c30f074164564084aed8122794b78746 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640' with partition [<start>, <end>)
00:28:40.797 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c30f074164564084aed8122794b78746@[2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999),731068d234f34dc09763e08ded790c5d(127.3.237.130:41841),f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)[L]]}]
00:28:40.798 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c30f074164564084aed8122794b78746, attempt=13, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2680), Traces: [0ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [0ms] delaying RPC due to: Illegal state: connection is terminated, [17ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [18ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [18ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [38ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [39ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [40ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [40ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [58ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [59ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [60ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [60ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [118ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [119ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [120ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [120ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [177ms] refreshing cache from master, [177ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [179ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [180ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [181ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [181ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [356ms] refreshing cache from master, [356ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [359ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, [360ms] sending RPC to server 2ddcbfc125f04fd592241ffd11b0dfb7, [361ms] received response from server 2ddcbfc125f04fd592241ffd11b0dfb7: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [361ms] delaying RPC due to: Network error: Failed to connect to peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999): Connection refused: /127.3.237.131:41999, [2676ms] refreshing cache from master, [2677ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.237.190:45669, [2679ms] Sub RPC GetTableLocations: received response from server master-127.3.237.190:45669: OK, deferred=Deferred@511931925(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f010836b7a8@5277db23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819) -> (continuation of Deferred@169246997 after retry RPC@940198371) -> (continuation of Deferred@183343277 after retry RPC@1614974274) -> (continuation of Deferred@110526129 after retry RPC@1917504444), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b15c46c -> wakeup thread Time-limited test -> (continuation of Deferred@689924384 after retry RPC@1280114379) -> (continuation of Deferred@2055003003 after retry RPC@894500819) -> (continuation of Deferred@169246997 after retry RPC@940198371) -> (continuation of Deferred@183343277 after retry RPC@1614974274) -> (continuation of Deferred@110526129 after retry RPC@1917504444))) after lookup
00:28:40.798 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] connecting to peer
00:28:40.799 [DEBUG - kudu-nio-6] (Connection.java:757) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] Successfully connected to peer
00:28:40.823 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x68f2ad07, L:/127.0.0.1:47546 - R:/127.3.237.129:43753] using TOKEN/null
00:28:40.839 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.3.237.131:41999
00:28:40.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:40.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.839740  4022 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbWsn41/build/debug/bin/kudu
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:40.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.938688  6416 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:40.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.938859  6416 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:40.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.938900  6416 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.942054  6416 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.942145  6416 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.237.131
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.945915  6416 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.237.148:44315
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.237.131:41999
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.237.131
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41753
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.237.190:45669,127.3.237.189:40855,127.3.237.188:36829
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6416
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.237.131
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/logs
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:03 UTC on bdcb31816ec0
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16840
00:28:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.946801  6416 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:40.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.947885  6416 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.954052  6422 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.953992  6424 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.954291  6416 server_base.cc:1061] running on GCE node
00:28:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.954288  6421 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.954711  6416 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.955193  6416 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:40.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.956332  6416 hybrid_clock.cc:648] HybridClock initialized: now 1783470520956312 us; error 37 us; skew 500 ppm
00:28:40.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.957907  6416 webserver.cc:534] Webserver started at http://127.3.237.131:41753/ using document root <none> and password file <none>
00:28:40.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.958364  6416 fs_manager.cc:362] Metadata directory not provided
00:28:40.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.958413  6416 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:40.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.961345  6416 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
00:28:40.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.962970  6430 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.963933  6416 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.964017  6416 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data,/tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ddcbfc125f04fd592241ffd11b0dfb7"
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:28:06 on dist-test-slave-q60k"
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.964339  6416 fs_report.cc:389] FS layout report
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/wal
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/data
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.973990  6416 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.974453  6416 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.974586  6416 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.975000  6416 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.976111  6437 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:28:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.980404  6416 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.980458  6416 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.005s	user 0.000s	sys 0.000s
00:28:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.980499  6416 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.981779  6416 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.981842  6416 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
00:28:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.982115  6437 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap starting.
00:28:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.993047  6416 rpc_server.cc:307] RPC server started. Bound to: 127.3.237.131:41999
00:28:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.993059  6547 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.237.131:41999 every 8 connection(s)
00:28:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.994336  6416 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbWsn41/test-tmp/mini-kudu-cluster7181131665880437709/ts-2/data/info.pb
00:28:40.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.995215  4022 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbWsn41/build/debug/bin/kudu as pid 6416
00:28:40.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:41.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.002099  6437 log.cc:826] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Log is configured to *not* fsync() on all Append() calls
00:28:41.004 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:41.008 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470486640, tablet=c30f074164564084aed8122794b78746, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.010422  6549 heartbeater.cc:344] Connected to a master server at 127.3.237.190:45669
00:28:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.010656  6549 heartbeater.cc:461] Registering TS with master...
00:28:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.010774  6548 heartbeater.cc:344] Connected to a master server at 127.3.237.188:36829
00:28:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.010860  6548 heartbeater.cc:461] Registering TS with master...
00:28:41.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.011234  6549 heartbeater.cc:507] Master 127.3.237.190:45669 requested a full tablet report, sending...
00:28:41.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.011234  6548 heartbeater.cc:507] Master 127.3.237.188:36829 requested a full tablet report, sending...
00:28:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.012992  4064 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.013043  4208 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.013903  4064 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.237.131:46309
00:28:41.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.016651  6550 heartbeater.cc:344] Connected to a master server at 127.3.237.189:40855
00:28:41.017 [DEBUG - main] (Connection.java:575) [peer master-127.3.237.189:40855(127.3.237.189:40855)] disconnecting while in state READY
00:28:41.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.016726  6550 heartbeater.cc:461] Registering TS with master...
00:28:41.018 [DEBUG - main] (Connection.java:575) [peer 2ddcbfc125f04fd592241ffd11b0dfb7(127.3.237.131:41999)] disconnecting while in state TERMINATED
00:28:41.018 [DEBUG - main] (Connection.java:575) [peer master-127.3.237.190:45669(127.3.237.190:45669)] disconnecting while in state READY
00:28:41.018 [DEBUG - main] (Connection.java:575) [peer 731068d234f34dc09763e08ded790c5d(127.3.237.130:41841)] disconnecting while in state TERMINATED
00:28:41.018 [DEBUG - main] (Connection.java:575) [peer master-127.3.237.188:36829(127.3.237.188:36829)] disconnecting while in state READY
00:28:41.018 [DEBUG - main] (Connection.java:575) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] disconnecting while in state READY
00:28:41.021 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.3.237.188:36829(127.3.237.188:36829)] handling channelInactive
00:28:41.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.237.188:36829(127.3.237.188:36829)] cleaning up while in state READY due to: connection closed
00:28:41.021 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.3.237.190:45669(127.3.237.190:45669)] handling channelInactive
00:28:41.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.237.190:45669(127.3.237.190:45669)] cleaning up while in state READY due to: connection closed
00:28:41.021 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.237.189:40855(127.3.237.189:40855)] handling channelInactive
00:28:41.022 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.237.189:40855(127.3.237.189:40855)] cleaning up while in state READY due to: connection closed
00:28:41.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.016880  6550 heartbeater.cc:507] Master 127.3.237.189:40855 requested a full tablet report, sending...
00:28:41.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.017911  4136 ts_manager.cc:194] Re-registered known tserver with Master: 2ddcbfc125f04fd592241ffd11b0dfb7 (127.3.237.131:41999)
00:28:41.023 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:28:41.027 [DEBUG - kudu-nio-6] (Connection.java:239) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] handling channelInactive
00:28:41.027 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f209795a44804bb3aea71582dc28cf0f(127.3.237.129:43753)] cleaning up while in state READY due to: connection closed
00:28:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.028409  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 6060
00:28:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.035598  6437 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:41.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.036069  6437 tablet_bootstrap.cc:492] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Bootstrap complete.
00:28:41.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.036609  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 6235
00:28:41.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.037575  6437 ts_tablet_manager.cc:1403] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent bootstrapping tablet: real 0.056s	user 0.037s	sys 0.011s
00:28:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.039680  6437 raft_consensus.cc:359] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.040092  6437 raft_consensus.cc:740] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ddcbfc125f04fd592241ffd11b0dfb7, State: Initialized, Role: FOLLOWER
00:28:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.040593  6437 consensus_queue.cc:260] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ddcbfc125f04fd592241ffd11b0dfb7" member_type: VOTER last_known_addr { host: "127.3.237.131" port: 41999 } } peers { permanent_uuid: "f209795a44804bb3aea71582dc28cf0f" member_type: VOTER last_known_addr { host: "127.3.237.129" port: 43753 } } peers { permanent_uuid: "731068d234f34dc09763e08ded790c5d" member_type: VOTER last_known_addr { host: "127.3.237.130" port: 41841 } }
00:28:41.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.041452  6437 ts_tablet_manager.cc:1434] T c30f074164564084aed8122794b78746 P 2ddcbfc125f04fd592241ffd11b0dfb7: Time spent starting tablet: real 0.004s	user 0.003s	sys 0.000s
00:28:41.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.041713  6549 heartbeater.cc:499] Master 127.3.237.190:45669 was elected leader, sending a full tablet report...
00:28:41.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.044448  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 6416
00:28:41.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.051009  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4033
00:28:41.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.057538  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4102
00:28:41.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.063386  4022 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbWsn41/build/debug/bin/kudu with pid 4174
00:28:41.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:28:41Z chronyd exiting

Time: 36.953

OK (1 test)

