JUnit version 4.13.2
.14:01:14.473 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783519274464
14:01:14.534 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
14:01:14.594 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:01:14.607 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktTzzjk/test-tmp
14:01:14.613 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
14:01:14.616 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktTzzjk/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
14:01:14.842 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

14:01:15.065 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
14:01:15.066 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

14:01:15.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.070354 14301 mini_oidc.cc:150] Starting JWKS server
14:01:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.073654 14301 webserver.cc:533] Webserver started at http://127.0.0.1:38525/ using document root <none> and password file <none>
14:01:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.074004 14301 mini_oidc.cc:186] Starting OIDC Discovery server
14:01:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.075462 14301 webserver.cc:533] Webserver started at http://127.0.0.1:42733/ using document root <none> and password file <none>
14:01:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T14:01:15Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:01:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T14:01:15Z Disabled control of system clock
14:01:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.109210 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/wal
14:01:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data
14:01:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/logs
14:01:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data/info.pb
14:01:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.126:34459
14:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.126
14:01:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:01:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:01:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38525/jwks.json with env {}
14:01:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.441187 14312 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.441505 14312 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.441586 14312 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.450466 14312 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:01:15.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.450565 14312 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:15.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.450620 14312 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:01:15.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.450660 14312 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:01:15.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.450716 14312 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:01:15.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.450749 14312 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38525/jwks.json
14:01:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.461988 14312 master_runner.cc:387] Master server non-default flags:
14:01:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:15.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:15.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:15.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data
14:01:15.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/wal
14:01:15.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:01:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:01:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.126:34459
14:01:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:01:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:15.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:01:15.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38525/jwks.json
14:01:15.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:15.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:15.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data/info.pb
14:01:15.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.126
14:01:15.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:15.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:15.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:15.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:15.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/logs
14:01:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:01:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.463927 14312 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.466028 14312 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.476558 14317 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.476567 14318 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.476558 14320 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.476809 14312 server_base.cc:1061] running on GCE node
14:01:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.478175 14312 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.479892 14312 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.481181 14312 hybrid_clock.cc:648] HybridClock initialized: now 1783519275481115 us; error 42 us; skew 500 ppm
14:01:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.481689 14312 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.484373 14312 webserver.cc:533] Webserver started at http://127.13.247.126:45669/ using document root <none> and password file <none>
14:01:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.485275 14312 fs_manager.cc:362] Metadata directory not provided
14:01:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.485419 14312 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.485896 14312 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:01:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.488631 14312 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data/instance:
14:01:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5241e7eb1f6646b7ae03f16c3358c2e3"
14:01:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:15 on dist-test-slave-f6s0"
14:01:15.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.489642 14312 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/wal/instance:
14:01:15.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5241e7eb1f6646b7ae03f16c3358c2e3"
14:01:15.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:15 on dist-test-slave-f6s0"
14:01:15.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.496070 14312 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.003s	sys 0.005s
14:01:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.499964 14326 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.501577 14312 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
14:01:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.501729 14312 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/wal
14:01:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5241e7eb1f6646b7ae03f16c3358c2e3"
14:01:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:15 on dist-test-slave-f6s0"
14:01:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.501911 14312 fs_report.cc:389] FS layout report
14:01:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/wal
14:01:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/wal
14:01:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data/data
14:01:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:15.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:15.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:15.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:15.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:15.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.529764 14312 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:15.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.530754 14312 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:15.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.531004 14312 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:15.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.552184 14312 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.126:34459
14:01:15.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.552229 14377 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.126:34459 every 8 connection(s)
14:01:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.553951 14312 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/data/info.pb
14:01:15.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.558089 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 14312
14:01:15.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.558432 14301 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-0/wal/instance
14:01:15.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.559387 14378 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:01:15.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.561564 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/wal
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:15.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:15.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/logs
14:01:15.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data/info.pb
14:01:15.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:01:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:01:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:01:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.125:33751
14:01:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.125
14:01:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:15.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:15.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:15.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:01:15.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:15.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:01:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38525/jwks.json with env {}
14:01:15.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.568322 14378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } has no permanent_uuid. Determining permanent_uuid...
14:01:15.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.586524 14378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } has no permanent_uuid. Determining permanent_uuid...
14:01:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.588037 14327 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.125:33751: connect: Connection refused (error 111)
14:01:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.589838 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.125:33751: Network error: Client connection negotiation failed: client connection to 127.13.247.125:33751: connect: Connection refused (error 111)
14:01:15.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.639341 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } attempt: 1
14:01:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.641857 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.125:33751: Network error: Client connection negotiation failed: client connection to 127.13.247.125:33751: connect: Connection refused (error 111)
14:01:15.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.710175 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } attempt: 2
14:01:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.713138 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.125:33751: Network error: Client connection negotiation failed: client connection to 127.13.247.125:33751: connect: Connection refused (error 111)
14:01:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.804428 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } attempt: 3
14:01:15.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.806912 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.125:33751: Network error: Client connection negotiation failed: client connection to 127.13.247.125:33751: connect: Connection refused (error 111)
14:01:15.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.909319 14380 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:15.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.909723 14380 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:15.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.909854 14380 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.918759 14380 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:01:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.918917 14380 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.919004 14380 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:01:15.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.919062 14380 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:01:15.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.919185 14380 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:01:15.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.919282 14380 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38525/jwks.json
14:01:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.929924 14380 master_runner.cc:387] Master server non-default flags:
14:01:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data
14:01:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/wal
14:01:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:01:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:01:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.125:33751
14:01:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:01:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:01:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38525/jwks.json
14:01:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data/info.pb
14:01:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.125
14:01:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/logs
14:01:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:01:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.932253 14380 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.934820 14380 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:15.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.944628 14389 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:15.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.944661 14388 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:15.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.948033 14391 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:15.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.948078 14380 server_base.cc:1061] running on GCE node
14:01:15.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.949568 14380 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:15.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.950179 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } attempt: 4
14:01:15.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:15.952767 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.125:33751: Network error: Client connection negotiation failed: client connection to 127.13.247.125:33751: connect: Connection refused (error 111)
14:01:15.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.953044 14380 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:15.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.954264 14380 hybrid_clock.cc:648] HybridClock initialized: now 1783519275954176 us; error 83 us; skew 500 ppm
14:01:15.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.954632 14380 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:15.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.957803 14380 webserver.cc:533] Webserver started at http://127.13.247.125:41955/ using document root <none> and password file <none>
14:01:15.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.958648 14380 fs_manager.cc:362] Metadata directory not provided
14:01:15.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.958745 14380 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:15.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.959178 14380 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:01:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.962612 14380 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data/instance:
14:01:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7787fcc61c5c41729622414f35ca3406"
14:01:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:15 on dist-test-slave-f6s0"
14:01:15.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.963716 14380 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/wal/instance:
14:01:15.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7787fcc61c5c41729622414f35ca3406"
14:01:15.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:15 on dist-test-slave-f6s0"
14:01:15.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.970527 14380 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
14:01:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.974692 14397 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.976369 14380 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
14:01:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.976529 14380 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/wal
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7787fcc61c5c41729622414f35ca3406"
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:15 on dist-test-slave-f6s0"
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.976740 14380 fs_report.cc:389] FS layout report
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/wal
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/wal
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data/data
14:01:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:15.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:15.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:15.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.990235 14380 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:15.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.991405 14380 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:15.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:15.991750 14380 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.013695 14380 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.125:33751
14:01:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.013715 14448 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.125:33751 every 8 connection(s)
14:01:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.015506 14380 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/data/info.pb
14:01:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.020190 14449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:01:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.020346 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 14380
14:01:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.020468 14301 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-1/wal/instance
14:01:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.021677 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/wal
14:01:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data
14:01:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/logs
14:01:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data/info.pb
14:01:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:01:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:01:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:01:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.124:36625
14:01:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.124
14:01:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:01:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:01:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38525/jwks.json with env {}
14:01:16.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.029271 14449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } has no permanent_uuid. Determining permanent_uuid...
14:01:16.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.047056 14449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } has no permanent_uuid. Determining permanent_uuid...
14:01:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.057142 14449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } has no permanent_uuid. Determining permanent_uuid...
14:01:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.058316 14399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.060027 14449 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.109412 14449 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 1
14:01:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.111732 14449 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.179949 14449 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 2
14:01:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.182024 14449 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.252149 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } attempt: 5
14:01:16.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.259418 14378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } has no permanent_uuid. Determining permanent_uuid...
14:01:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.261536 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.273265 14449 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 3
14:01:16.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.275470 14449 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.312796 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 1
14:01:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.315161 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.376310 14451 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:16.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.376670 14451 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:16.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.376753 14451 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.383409 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 2
14:01:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.385654 14378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.387380 14451 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:01:16.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.387466 14451 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.387523 14451 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:01:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.387596 14451 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:01:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.387662 14451 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:01:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.387696 14451 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38525/jwks.json
14:01:16.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.398504 14451 master_runner.cc:387] Master server non-default flags:
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/wal
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:01:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.124:36625
14:01:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:01:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:01:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38525/jwks.json
14:01:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data/info.pb
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.124
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:16.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/logs
14:01:16.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:16.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:16.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:16.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:01:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.400328 14451 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.402259 14451 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:16.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.411054 14460 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:16.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.411067 14459 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:16.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.412343 14462 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.412829 14451 server_base.cc:1061] running on GCE node
14:01:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.413590 14451 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:16.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.415118 14451 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.416395 14451 hybrid_clock.cc:648] HybridClock initialized: now 1783519276416324 us; error 60 us; skew 500 ppm
14:01:16.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.416837 14451 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:16.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.418725 14449 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 4
14:01:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.419627 14451 webserver.cc:533] Webserver started at http://127.13.247.124:35401/ using document root <none> and password file <none>
14:01:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.420681 14451 fs_manager.cc:362] Metadata directory not provided
14:01:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.420810 14451 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.420790 14449 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.247.124:36625: Network error: Client connection negotiation failed: client connection to 127.13.247.124:36625: connect: Connection refused (error 111)
14:01:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.421195 14451 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:01:16.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.423674 14451 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data/instance:
14:01:16.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4aa37101a3d24b40a25ddb6a104d7ced"
14:01:16.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:16 on dist-test-slave-f6s0"
14:01:16.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.424453 14451 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/wal/instance:
14:01:16.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4aa37101a3d24b40a25ddb6a104d7ced"
14:01:16.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:16 on dist-test-slave-f6s0"
14:01:16.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.429900 14451 fs_manager.cc:696] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.000s
14:01:16.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.433467 14468 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:16.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.435039 14451 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
14:01:16.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.435283 14451 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/wal
14:01:16.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4aa37101a3d24b40a25ddb6a104d7ced"
14:01:16.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:16 on dist-test-slave-f6s0"
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.435597 14451 fs_report.cc:389] FS layout report
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/wal
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/wal
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data/data
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:16.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.457191 14451 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.458424 14451 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.458801 14451 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.482443 14519 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.124:36625 every 8 connection(s)
14:01:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.482502 14451 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.124:36625
14:01:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.484642 14451 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/data/info.pb
14:01:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.489362 14520 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:01:16.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.491887 14378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 3
14:01:16.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.492079 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 14451
14:01:16.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.492256 14301 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/master-2/wal/instance
14:01:16.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.496735 14520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } has no permanent_uuid. Determining permanent_uuid...
14:01:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.511411 14520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } has no permanent_uuid. Determining permanent_uuid...
14:01:16.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.514956 14378 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: Bootstrap starting.
14:01:16.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.519196 14520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } has no permanent_uuid. Determining permanent_uuid...
14:01:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.520283 14378 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: Neither blocks nor log segments found. Creating new log.
14:01:16.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.522166 14378 log.cc:826] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: Log is configured to *not* fsync() on all Append() calls
14:01:16.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.525808 14378 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: No bootstrap required, opened a new log
14:01:16.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.530938 14520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced: Bootstrap starting.
14:01:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.532151 14378 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.532403 14378 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:01:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.532482 14378 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5241e7eb1f6646b7ae03f16c3358c2e3, State: Initialized, Role: FOLLOWER
14:01:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.533128 14378 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.534286 14529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.534863 14529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [sys.catalog]: This master's current role is: FOLLOWER
14:01:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.534896 14520 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced: Neither blocks nor log segments found. Creating new log.
14:01:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.535648 14378 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [sys.catalog]: configured and running, proceeding with master startup.
14:01:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.536219 14520 log.cc:826] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced: Log is configured to *not* fsync() on all Append() calls
14:01:16.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.538859 14520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced: No bootstrap required, opened a new log
14:01:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.545847 14520 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.546748 14520 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:01:16.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.547196 14520 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4aa37101a3d24b40a25ddb6a104d7ced, State: Initialized, Role: FOLLOWER
14:01:16.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.548009 14520 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.549808 14543 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.550843 14543 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: This master's current role is: FOLLOWER
14:01:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.553531 14520 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: configured and running, proceeding with master startup.
14:01:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.557425 14541 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:01:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.557550 14541 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:01:16.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.548907 14312 master_runner.cc:187] Error getting master registration for 127.13.247.125:33751: OK, instance_id { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" instance_seqno: 1783519276007842 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:01:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.558887 14312 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:01:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.579778 14554 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:01:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.580497 14554 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:01:16.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.579560 14451 master_runner.cc:187] Error getting master registration for 127.13.247.125:33751: OK, instance_id { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" instance_seqno: 1783519276007842 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:01:16.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.586249 14451 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:01:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.666257 14543 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:01:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.666560 14543 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.668583 14543 leader_election.cc:290] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5241e7eb1f6646b7ae03f16c3358c2e3 (127.13.247.126:34459), 7787fcc61c5c41729622414f35ca3406 (127.13.247.125:33751)
14:01:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.669338 14353 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" is_pre_election: true
14:01:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.669857 14353 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4aa37101a3d24b40a25ddb6a104d7ced in term 0.
14:01:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.671434 14471 leader_election.cc:304] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4aa37101a3d24b40a25ddb6a104d7ced, 5241e7eb1f6646b7ae03f16c3358c2e3; no voters: 
14:01:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.671980 14543 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Leader pre-election won for term 1
14:01:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.672119 14543 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:01:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.672216 14543 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 0 FOLLOWER]: Advancing to term 1
14:01:16.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.675732 14543 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.677000 14353 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3"
14:01:16.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.677255 14353 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [term 0 FOLLOWER]: Advancing to term 1
14:01:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.677619 14543 leader_election.cc:290] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [CANDIDATE]: Term 1 election: Requested vote from peers 5241e7eb1f6646b7ae03f16c3358c2e3 (127.13.247.126:34459), 7787fcc61c5c41729622414f35ca3406 (127.13.247.125:33751)
14:01:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.685106 14353 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4aa37101a3d24b40a25ddb6a104d7ced in term 1.
14:01:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.688441 14471 leader_election.cc:304] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4aa37101a3d24b40a25ddb6a104d7ced, 5241e7eb1f6646b7ae03f16c3358c2e3; no voters: 
14:01:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.688843 14543 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 1 FOLLOWER]: Leader election won for term 1
14:01:16.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.694432 14543 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [term 1 LEADER]: Becoming Leader. State: Replica: 4aa37101a3d24b40a25ddb6a104d7ced, State: Running, Role: LEADER
14:01:16.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.695088 14543 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.699671 14556 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4aa37101a3d24b40a25ddb6a104d7ced. Latest consensus state: current_term: 1 leader_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.700042 14556 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: This master's current role is: LEADER
14:01:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.704259 14558 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
14:01:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.706748 14558 catalog_manager.cc:1520] Initializing Kudu cluster ID...
14:01:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:16.711134 14474 tablet.cc:2406] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:01:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.719977 14449 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } attempt: 5
14:01:16.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.723731 14353 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [term 1 FOLLOWER]: Refusing update from remote peer 4aa37101a3d24b40a25ddb6a104d7ced: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:01:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.725507 14543 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [LEADER]: Connected to new peer: Peer: permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:01:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.736905 14449 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406: Bootstrap starting.
14:01:16.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.743039 14449 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406: Neither blocks nor log segments found. Creating new log.
14:01:16.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.751809 14449 log.cc:826] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406: Log is configured to *not* fsync() on all Append() calls
14:01:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.764297 14529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4aa37101a3d24b40a25ddb6a104d7ced. Latest consensus state: current_term: 1 leader_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.764571 14529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [sys.catalog]: This master's current role is: FOLLOWER
14:01:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.771195 14559 mvcc.cc:204] Tried to move back new op lower bound from 7305294957444018176 to 7305294957349564416. Current Snapshot: MvccSnapshot[applied={T|T < 7305294957444018176}]
14:01:16.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.781172 14449 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406: No bootstrap required, opened a new log
14:01:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.783524 14543 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.784844 14543 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: This master's current role is: LEADER
14:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.789963 14529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.790153 14529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3 [sys.catalog]: This master's current role is: FOLLOWER
14:01:16.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.800858 14449 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.801245 14449 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:01:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.801367 14449 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7787fcc61c5c41729622414f35ca3406, State: Initialized, Role: FOLLOWER
14:01:16.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.803653 14558 catalog_manager.cc:1383] Generated new cluster ID: f302b1bdd81f4007b298833f201c828d
14:01:16.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.803779 14558 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
14:01:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.804389 14449 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.825246 14565 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.827979 14565 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [sys.catalog]: This master's current role is: FOLLOWER
14:01:16.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.832835 14558 catalog_manager.cc:1406] Generated new certificate authority record
14:01:16.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.834501 14558 catalog_manager.cc:1540] Loading token signing keys...
14:01:16.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.832500 14424 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7787fcc61c5c41729622414f35ca3406" is_pre_election: true
14:01:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.833250 14422 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 0 FOLLOWER]: Advancing to term 1
14:01:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.840996 14422 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 1 FOLLOWER]: Refusing update from remote peer 4aa37101a3d24b40a25ddb6a104d7ced: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:01:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.833400 14423 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7787fcc61c5c41729622414f35ca3406"
14:01:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.842061 14556 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [LEADER]: Connected to new peer: Peer: permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:01:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.846060 14423 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4aa37101a3d24b40a25ddb6a104d7ced in term 1.
14:01:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.833659 14449 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [sys.catalog]: configured and running, proceeding with master startup.
14:01:16.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.879201 14558 catalog_manager.cc:6070] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced: Generated new TSK 0
14:01:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.880950 14558 catalog_manager.cc:1550] Initializing in-progress tserver states...
14:01:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.890491 14556 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.890398 14566 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4aa37101a3d24b40a25ddb6a104d7ced. Latest consensus state: current_term: 1 leader_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.890764 14556 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4aa37101a3d24b40a25ddb6a104d7ced [sys.catalog]: This master's current role is: LEADER
14:01:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.890777 14566 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [sys.catalog]: This master's current role is: FOLLOWER
14:01:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.891191 14567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } } }
14:01:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.891558 14567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [sys.catalog]: This master's current role is: FOLLOWER
14:01:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.891737 14565 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 4aa37101a3d24b40a25ddb6a104d7ced)
14:01:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.891911 14565 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" member_type: VOTER last_known_addr { host: "127.13.247.126" port: 34459 } } peers { permanent_uuid: "7787fcc61c5c41729622414f35ca3406" member_type: VOTER last_known_addr { host: "127.13.247.125" port: 33751 } } peers { permanent_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" member_type: VOTER last_known_addr { host: "127.13.247.124" port: 36625 } }
14:01:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.893976 14565 leader_election.cc:290] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5241e7eb1f6646b7ae03f16c3358c2e3 (127.13.247.126:34459), 4aa37101a3d24b40a25ddb6a104d7ced (127.13.247.124:36625)
14:01:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.894479 14353 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7787fcc61c5c41729622414f35ca3406" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: false dest_uuid: "5241e7eb1f6646b7ae03f16c3358c2e3" is_pre_election: true
14:01:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.895061 14495 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7787fcc61c5c41729622414f35ca3406" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: false dest_uuid: "4aa37101a3d24b40a25ddb6a104d7ced" is_pre_election: true
14:01:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.896212 14400 leader_election.cc:304] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7787fcc61c5c41729622414f35ca3406; no voters: 4aa37101a3d24b40a25ddb6a104d7ced, 5241e7eb1f6646b7ae03f16c3358c2e3
14:01:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.898936 14565 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:01:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.898968 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:0
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:16.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:16.986018 14582 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:01:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.314913 14576 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.315271 14576 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.315356 14576 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.323926 14576 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.324087 14576 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.65
14:01:17.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.334621 14576 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:0
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.336412 14576 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.338510 14576 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.349280 14587 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:17.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.349746 14588 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:17.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.350670 14590 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.352352 14576 server_base.cc:1061] running on GCE node
14:01:17.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.353173 14576 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.354364 14576 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:17.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.355577 14576 hybrid_clock.cc:648] HybridClock initialized: now 1783519277355470 us; error 110 us; skew 500 ppm
14:01:17.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.355906 14576 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.358351 14576 webserver.cc:533] Webserver started at http://127.13.247.65:32823/ using document root <none> and password file <none>
14:01:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.359122 14576 fs_manager.cc:362] Metadata directory not provided
14:01:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.359304 14576 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.359726 14576 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:01:17.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.362114 14576 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/instance:
14:01:17.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:17.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.362936 14576 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal/instance:
14:01:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:17.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.370684 14576 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.001s
14:01:17.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.374909 14596 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.376850 14576 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.377080 14576 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.377416 14576 fs_report.cc:389] FS layout report
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/data
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.406818 14576 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:17.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.407954 14576 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:17.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.408319 14576 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:17.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.409587 14576 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.411528 14576 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:01:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.411657 14576 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:01:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.411763 14576 ts_tablet_manager.cc:616] Registered 0 tablets
14:01:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.411849 14576 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:01:17.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.449256 14576 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.65:44239
14:01:17.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.449316 14708 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.65:44239 every 8 connection(s)
14:01:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.451002 14576 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.454843 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 14576
14:01:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.455103 14301 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal/instance
14:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.473487 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:0
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.478088 14710 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.478624 14710 heartbeater.cc:461] Registering TS with master...
14:01:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.480088 14710 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.483167 14711 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.483392 14711 heartbeater.cc:461] Registering TS with master...
14:01:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.483970 14711 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.483963 14343 ts_manager.cc:194] Registered new tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.485088 14709 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.485438 14709 heartbeater.cc:461] Registering TS with master...
14:01:17.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.485852 14414 ts_manager.cc:194] Registered new tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.486198 14709 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.488631 14485 ts_manager.cc:194] Registered new tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.490871 14485 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.65:55919
14:01:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.559811 14541 catalog_manager.cc:1295] Loaded cluster ID: f302b1bdd81f4007b298833f201c828d
14:01:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.559953 14541 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: loading cluster ID for follower catalog manager: success
14:01:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.563057 14541 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: acquiring CA information for follower catalog manager: success
14:01:17.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.565119 14541 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 5241e7eb1f6646b7ae03f16c3358c2e3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:01:17.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.824502 14719 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.824844 14719 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.824926 14719 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:17.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.833489 14719 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:17.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.833664 14719 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.66
14:01:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.844501 14719 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:0
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.846146 14719 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.848043 14719 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:17.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.857892 14724 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:17.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.857923 14725 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:17.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:17.858640 14727 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:17.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.859009 14719 server_base.cc:1061] running on GCE node
14:01:17.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.859756 14719 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:17.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.861013 14719 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:17.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.862285 14719 hybrid_clock.cc:648] HybridClock initialized: now 1783519277862209 us; error 63 us; skew 500 ppm
14:01:17.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.862658 14719 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.865653 14719 webserver.cc:533] Webserver started at http://127.13.247.66:36973/ using document root <none> and password file <none>
14:01:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.866624 14719 fs_manager.cc:362] Metadata directory not provided
14:01:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.866726 14719 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.867026 14719 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.870015 14719 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/instance:
14:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:17.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.870827 14719 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal/instance:
14:01:17.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:17.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.876933 14719 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.001s
14:01:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.880844 14733 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.882786 14719 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.882936 14719 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.883106 14719 fs_report.cc:389] FS layout report
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/data
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:17.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.914465 14719 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.915431 14719 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.915686 14719 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:17.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.916710 14719 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.918591 14719 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:01:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.918684 14719 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:01:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.918761 14719 ts_tablet_manager.cc:616] Registered 0 tablets
14:01:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.918819 14719 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:01:17.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.954958 14719 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.66:46279
14:01:17.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.955026 14845 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.66:46279 every 8 connection(s)
14:01:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.956588 14719 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.962801 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 14719
14:01:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.963037 14301 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal/instance
14:01:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.967191 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:0
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.977159 14846 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.977572 14846 heartbeater.cc:461] Registering TS with master...
14:01:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.978739 14846 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.979337 14847 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.979516 14847 heartbeater.cc:461] Registering TS with master...
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.980074 14847 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.980377 14848 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.980512 14848 heartbeater.cc:461] Registering TS with master...
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.980952 14848 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.981276 14343 ts_manager.cc:194] Registered new tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.981559 14485 ts_manager.cc:194] Registered new tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.982086 14414 ts_manager.cc:194] Registered new tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.982681 14485 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.66:50399
14:01:17.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.991801 14582 catalog_manager.cc:1295] Loaded cluster ID: f302b1bdd81f4007b298833f201c828d
14:01:17.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.991927 14582 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406: loading cluster ID for follower catalog manager: success
14:01:17.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.995982 14582 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406: acquiring CA information for follower catalog manager: success
14:01:17.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:17.998834 14582 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 7787fcc61c5c41729622414f35ca3406: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:01:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.334177 14853 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.334508 14853 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.334590 14853 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:18.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.343572 14853 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.343739 14853 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.67
14:01:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.354399 14853 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:0
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.356156 14853 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.358331 14853 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.368358 14859 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:18.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.368533 14860 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:18.371683 14862 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.372409 14853 server_base.cc:1061] running on GCE node
14:01:18.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.373034 14853 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.374056 14853 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.375420 14853 hybrid_clock.cc:648] HybridClock initialized: now 1783519278375331 us; error 71 us; skew 500 ppm
14:01:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.375728 14853 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.378641 14853 webserver.cc:533] Webserver started at http://127.13.247.67:43327/ using document root <none> and password file <none>
14:01:18.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.379571 14853 fs_manager.cc:362] Metadata directory not provided
14:01:18.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.379678 14853 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:18.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.380190 14853 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:01:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.383116 14853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/instance:
14:01:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:18 on dist-test-slave-f6s0"
14:01:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.384163 14853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal/instance:
14:01:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:18 on dist-test-slave-f6s0"
14:01:18.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.390301 14853 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
14:01:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.393929 14868 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:18.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.395484 14853 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
14:01:18.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.395622 14853 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:18.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:18.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:18 on dist-test-slave-f6s0"
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.395831 14853 fs_report.cc:389] FS layout report
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/data
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:18.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.408061 14853 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:18.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.409008 14853 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:18.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.409309 14853 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:18.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.410235 14853 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.411944 14853 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:01:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.412030 14853 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:01:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.412101 14853 ts_tablet_manager.cc:616] Registered 0 tablets
14:01:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.412150 14853 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:01:18.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.448386 14853 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.67:36071
14:01:18.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.448459 14980 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.67:36071 every 8 connection(s)
14:01:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.450736 14853 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.459738 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 14853
14:01:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.459939 14301 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal/instance
14:01:18.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.476925 14981 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:18.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.477264 14981 heartbeater.cc:461] Registering TS with master...
14:01:18.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.478084 14981 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.478793 14983 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.478983 14983 heartbeater.cc:461] Registering TS with master...
14:01:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.479336 14982 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.479525 14982 heartbeater.cc:461] Registering TS with master...
14:01:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.479580 14983 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.479730 14485 ts_manager.cc:194] Registered new tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.480072 14982 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.480756 14485 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.67:60669
14:01:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.480782 14414 ts_manager.cc:194] Registered new tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.481526 14343 ts_manager.cc:194] Registered new tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.491137 14301 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
14:01:18.493 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
14:01:18.494 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

14:01:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.495117 14709 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:18.503 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.13.247.126"
      port: 34459
    }
    bound_http_address {
      host: "127.13.247.126"
      port: 45669
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.13.247.125"
      port: 33751
    }
    bound_http_address {
      host: "127.13.247.125"
      port: 41955
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.13.247.124"
      port: 36625
    }
    bound_http_address {
      host: "127.13.247.124"
      port: 35401
    }
  }
}

14:01:18.507 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

14:01:18.510 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.13.247.65"
      port: 44239
    }
    bound_http_address {
      host: "127.13.247.65"
      port: 32823
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.13.247.66"
      port: 46279
    }
    bound_http_address {
      host: "127.13.247.66"
      port: 36973
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.13.247.67"
      port: 36071
    }
    bound_http_address {
      host: "127.13.247.67"
      port: 43327
    }
  }
}

14:01:18.510 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:01:18.677 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:01:18.740 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
14:01:18.888 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.247.124:36625(127.13.247.124:36625)] connecting to peer
14:01:18.935 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.247.126:34459(127.13.247.126:34459)] connecting to peer
14:01:18.937 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.247.125:33751(127.13.247.125:33751)] connecting to peer
14:01:18.956 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.13.247.125:33751(127.13.247.125:33751)] Successfully connected to peer
14:01:18.956 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.247.126:34459(127.13.247.126:34459)] Successfully connected to peer
14:01:18.959 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.247.124:36625(127.13.247.124:36625)] Successfully connected to peer
14:01:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:18.985824 14846 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:19.067 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.13.247.126
14:01:19.069 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.13.247.125
14:01:19.067 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.13.247.124
14:01:19.294 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x835b068a, L:/127.0.0.1:38030 - R:/127.13.247.124:36625] using SASL/PLAIN
14:01:19.294 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x30df9f5a, L:/127.0.0.1:38864 - R:/127.13.247.126:34459] using SASL/PLAIN
14:01:19.294 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xd1106335, L:/127.0.0.1:42100 - R:/127.13.247.125:33751] using SASL/PLAIN
14:01:19.330 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:01:19.331 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=479), Traces: [0ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.247.124:36625, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.247.126:34459, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.247.125:33751, [453ms] Sub RPC ConnectToMaster: received response from server master-127.13.247.126:34459: OK, [454ms] Sub RPC ConnectToMaster: received response from server master-127.13.247.125:33751: OK, [454ms] Sub RPC ConnectToMaster: received response from server master-127.13.247.124:36625: OK, deferred=null) after lookup
14:01:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.378373 14485 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38030:
14:01:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840"
14:01:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:01:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:01:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
14:01:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:01:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:01:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:01:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:19.380430 14485 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:01:19.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.466391 14644 tablet_service.cc:1511] Processing CreateTablet for tablet 48add68e49ac4613aed5d6771722e462 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840 [id=2f4eed8fc7fb44c494cc87a87c7aad69]), partition=RANGE (key) PARTITION UNBOUNDED
14:01:19.468 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 2f4eed8fc7fb44c494cc87a87c7aad69
14:01:19.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.468394 14916 tablet_service.cc:1511] Processing CreateTablet for tablet 48add68e49ac4613aed5d6771722e462 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840 [id=2f4eed8fc7fb44c494cc87a87c7aad69]), partition=RANGE (key) PARTITION UNBOUNDED
14:01:19.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.469807 14916 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48add68e49ac4613aed5d6771722e462. 1 dirs total, 0 dirs full, 0 dirs failed
14:01:19.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.469808 14644 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48add68e49ac4613aed5d6771722e462. 1 dirs total, 0 dirs full, 0 dirs failed
14:01:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.472040 14781 tablet_service.cc:1511] Processing CreateTablet for tablet 48add68e49ac4613aed5d6771722e462 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840 [id=2f4eed8fc7fb44c494cc87a87c7aad69]), partition=RANGE (key) PARTITION UNBOUNDED
14:01:19.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.474990 14781 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48add68e49ac4613aed5d6771722e462. 1 dirs total, 0 dirs full, 0 dirs failed
14:01:19.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.484318 14981 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:19.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.505885 15013 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap starting.
14:01:19.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.512207 15013 tablet_bootstrap.cc:654] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Neither blocks nor log segments found. Creating new log.
14:01:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.520027 15012 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap starting.
14:01:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.520716 15011 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap starting.
14:01:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.521320 15013 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Log is configured to *not* fsync() on all Append() calls
14:01:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.524801 15011 tablet_bootstrap.cc:654] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Neither blocks nor log segments found. Creating new log.
14:01:19.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.524812 15012 tablet_bootstrap.cc:654] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Neither blocks nor log segments found. Creating new log.
14:01:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.526283 15012 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Log is configured to *not* fsync() on all Append() calls
14:01:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.526399 15011 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Log is configured to *not* fsync() on all Append() calls
14:01:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.534262 15011 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: No bootstrap required, opened a new log
14:01:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.534947 15011 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent bootstrapping tablet: real 0.015s	user 0.013s	sys 0.000s
14:01:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.535151 15013 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: No bootstrap required, opened a new log
14:01:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.535686 15013 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent bootstrapping tablet: real 0.030s	user 0.018s	sys 0.002s
14:01:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.540153 15012 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: No bootstrap required, opened a new log
14:01:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.540716 15012 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent bootstrapping tablet: real 0.021s	user 0.010s	sys 0.004s
14:01:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.543735 15011 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.544273 15011 raft_consensus.cc:385] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:01:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.544397 15011 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Initialized, Role: FOLLOWER
14:01:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.545378 15011 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.545691 15013 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.546271 15013 raft_consensus.cc:385] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:01:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.546873 15013 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Initialized, Role: FOLLOWER
14:01:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.547818 15013 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.548208 15012 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.548899 15012 raft_consensus.cc:385] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:01:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.549223 15012 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bde3d574cb824a4aa08d2b3f34754f99, State: Initialized, Role: FOLLOWER
14:01:19.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.551065 15012 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.554311 15011 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent starting tablet: real 0.019s	user 0.014s	sys 0.003s
14:01:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.570881 15013 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent starting tablet: real 0.033s	user 0.021s	sys 0.005s
14:01:19.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.580296 15012 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent starting tablet: real 0.039s	user 0.020s	sys 0.015s
14:01:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:19.706233 14984 tablet.cc:2406] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:01:19.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:19.711556 14849 tablet.cc:2406] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:01:19.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:19.713513 14712 tablet.cc:2406] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:01:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.766845 15017 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:01:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.767210 15017 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.769253 15017 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.779264 14936 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082" is_pre_election: true
14:01:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.779477 14801 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99" is_pre_election: true
14:01:19.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.779904 14801 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 0.
14:01:19.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.779904 14936 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 0.
14:01:19.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.780634 14598 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 
14:01:19.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.781028 15017 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:01:19.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.781194 15017 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:01:19.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.781296 15017 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 0 FOLLOWER]: Advancing to term 1
14:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.783704 15017 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.784435 15017 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 1 election: Requested vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.784905 14936 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.785168 14801 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.785168 14936 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 0 FOLLOWER]: Advancing to term 1
14:01:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.785382 14801 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 0 FOLLOWER]: Advancing to term 1
14:01:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.789005 14936 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 1.
14:01:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.789034 14801 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 1.
14:01:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.789580 14598 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 
14:01:19.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.790243 15017 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Leader election won for term 1
14:01:19.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.790791 15017 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 LEADER]: Becoming Leader. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Running, Role: LEADER
14:01:19.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.791462 15017 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:19.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:19.797175 14485 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 reported cstate change: term changed from 0 to 1, leader changed from <none> to fafc4088d51349e7973eb5b714841aa8 (127.13.247.65). New cstate: current_term: 1 leader_uuid: "fafc4088d51349e7973eb5b714841aa8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:19.882 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 2f4eed8fc7fb44c494cc87a87c7aad69
14:01:19.900 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:19.909 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299991, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:19.910 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [9ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=null) after lookup
14:01:19.924 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:19.926 [DEBUG - kudu-nio-3] (Connection.java:757) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] Successfully connected to peer
14:01:20.010 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0xce076332, L:/127.0.0.1:60498 - R:/127.13.247.65:44239] using TOKEN/null
14:01:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.039247 14936 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 1 FOLLOWER]: Refusing update from remote peer fafc4088d51349e7973eb5b714841aa8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:01:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.039247 14801 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 1 FOLLOWER]: Refusing update from remote peer fafc4088d51349e7973eb5b714841aa8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:01:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.040604 15022 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:01:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.040897 15017 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:01:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.068858 15026 mvcc.cc:204] Tried to move back new op lower bound from 7305294971027259392 to 7305294970029625344. Current Snapshot: MvccSnapshot[applied={T|T < 7305294971027259392}]
14:01:20.185 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:20.187 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.65:44239
14:01:20.188 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:20.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.189384 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 14576
14:01:20.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.221967 14704 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:20.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:20.328109 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 14576
14:01:20.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:20.352 [DEBUG - kudu-nio-3] (Connection.java:239) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] handling channelInactive
14:01:20.353 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state READY due to: connection closed
14:01:20.353 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: connection closed
14:01:20.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:20.378 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:20.379 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [5ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@454220617(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@4fccc002 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ee7e2a4 -> wakeup thread Time-limited test)) after lookup
14:01:20.380 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:20.382 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.382 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.383 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:20.397 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:20.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [5ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [34ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [35ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@454220617(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@4fccc002 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ee7e2a4 -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745))) after lookup
14:01:20.399 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:20.402 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.402 [DEBUG - kudu-nio-5] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.402 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.437 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:20.438 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:20.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=90), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [5ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [34ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [35ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [51ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [54ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [54ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@454220617(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@4fccc002 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ee7e2a4 -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130))) after lookup
14:01:20.439 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:20.441 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.441 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.441 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.517 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:20.517 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:20.518 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=169), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [5ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [34ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [35ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [51ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [54ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [54ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [91ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [93ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [93ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [165ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [169ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@454220617(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@4fccc002 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ee7e2a4 -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843))) after lookup
14:01:20.519 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:20.521 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.521 [DEBUG - kudu-nio-7] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.521 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:20.557 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:20.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=209), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [5ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [34ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [35ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [51ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [54ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [54ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [91ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [93ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [93ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [165ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [169ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [171ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [173ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [173ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [208ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@454220617(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@4fccc002 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843) -> (continuation of Deferred@597446498 after retry RPC@1880755379), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ee7e2a4 -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843) -> (continuation of Deferred@597446498 after retry RPC@1880755379))) after lookup
14:01:20.559 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:20.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.560 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.958 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:20.959 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:20.962 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=612), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [5ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [34ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [35ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [51ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [54ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [54ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [91ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [93ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [93ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [165ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [169ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [171ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [173ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [173ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [208ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [210ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [212ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [212ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [606ms] refreshing cache from master, [606ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [610ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@454220617(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@4fccc002 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843) -> (continuation of Deferred@597446498 after retry RPC@1880755379) -> (continuation of Deferred@706675122 after retry RPC@1130587628), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ee7e2a4 -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843) -> (continuation of Deferred@597446498 after retry RPC@1880755379) -> (continuation of Deferred@706675122 after retry RPC@1130587628))) after lookup
14:01:20.962 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:20.963 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.963 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:20.964 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.626646 15042 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.626986 15042 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.627187 15043 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.627410 15043 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:21.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.628883 15042 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:21.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.628968 15043 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:21.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:21.630080 14735 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:21.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:21.639969 14870 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.640753 14936 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082" is_pre_election: true
14:01:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.641013 14936 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bde3d574cb824a4aa08d2b3f34754f99 in term 1.
14:01:21.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.641623 14737 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: 
14:01:21.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.642061 15042 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:01:21.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.642215 15042 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 1 FOLLOWER]: Starting leader election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:21.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.642357 15042 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 1 FOLLOWER]: Advancing to term 2
14:01:21.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.642637 14801 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99" is_pre_election: true
14:01:21.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:21.642203 14735 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:21.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:21.643302 14870 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:21.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.644871 15042 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:21.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.645254 14801 raft_consensus.cc:2393] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 27ce0884bf764cdbb025c6d3b9edc082 in current term 2: Already voted for candidate bde3d574cb824a4aa08d2b3f34754f99 in this term.
14:01:21.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.645442 15042 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 2 election: Requested vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:21.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.646158 14936 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:21.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.646378 14936 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 1 FOLLOWER]: Advancing to term 2
14:01:21.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.646389 14870 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082; no voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8
14:01:21.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:21.647701 14735 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:21.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.648839 14936 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bde3d574cb824a4aa08d2b3f34754f99 in term 2.
14:01:21.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.649055 15043 raft_consensus.cc:2749] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:01:21.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.649288 14737 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: fafc4088d51349e7973eb5b714841aa8
14:01:21.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.649582 15042 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 FOLLOWER]: Leader election won for term 2
14:01:21.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.650049 15042 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 LEADER]: Becoming Leader. State: Replica: bde3d574cb824a4aa08d2b3f34754f99, State: Running, Role: LEADER
14:01:21.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.650671 15042 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:21.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:21.654721 14485 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 reported cstate change: term changed from 1 to 2, leader changed from fafc4088d51349e7973eb5b714841aa8 (127.13.247.65) to bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66). New cstate: current_term: 2 leader_uuid: "bde3d574cb824a4aa08d2b3f34754f99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: HEALTHY } } }
14:01:22.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:22.203768 14936 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Refusing update from remote peer bde3d574cb824a4aa08d2b3f34754f99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:01:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:22.204679 15065 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:01:22.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:22.214556 14735 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 -> Peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Couldn't send request to peer fafc4088d51349e7973eb5b714841aa8. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:22.957 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:22.958 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:22.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2610), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [5ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [34ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [35ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [51ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [54ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [54ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [91ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [93ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [93ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [165ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [169ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [171ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [173ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [173ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [208ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [210ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [212ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [212ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [606ms] refreshing cache from master, [606ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [610ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [614ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [616ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [616ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2605ms] refreshing cache from master, [2605ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [2609ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@454220617(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@4fccc002 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843) -> (continuation of Deferred@597446498 after retry RPC@1880755379) -> (continuation of Deferred@706675122 after retry RPC@1130587628) -> (continuation of Deferred@1313037796 after retry RPC@374803909), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ee7e2a4 -> wakeup thread Time-limited test -> (continuation of Deferred@1676444492 after retry RPC@2055498745) -> (continuation of Deferred@631119940 after retry RPC@1456263130) -> (continuation of Deferred@1026444088 after retry RPC@769626843) -> (continuation of Deferred@597446498 after retry RPC@1880755379) -> (continuation of Deferred@706675122 after retry RPC@1130587628) -> (continuation of Deferred@1313037796 after retry RPC@374803909))) after lookup
14:01:22.960 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:22.961 [DEBUG - kudu-nio-2] (Connection.java:757) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] Successfully connected to peer
14:01:22.987 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x0ff95c87, L:/127.0.0.1:58408 - R:/127.13.247.66:46279] using TOKEN/null
14:01:23.020 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.65:44239
14:01:23.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:23.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.023590 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:23.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:23.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:23.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:23.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:23.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:23.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.370376 15083 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:23.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.370697 15083 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:23.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.370795 15083 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.379586 15083 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.379805 15083 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.65
14:01:23.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.396543 15083 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:23.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.398429 15083 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.400985 15083 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:23.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.412339 15088 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.412708 15091 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.412401 15089 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.413599 15083 server_base.cc:1061] running on GCE node
14:01:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.414287 15083 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.415459 15083 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.416705 15083 hybrid_clock.cc:648] HybridClock initialized: now 1783519283416638 us; error 54 us; skew 500 ppm
14:01:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.417124 15083 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.419730 15083 webserver.cc:533] Webserver started at http://127.13.247.65:32823/ using document root <none> and password file <none>
14:01:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.420619 15083 fs_manager.cc:362] Metadata directory not provided
14:01:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.420790 15083 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:23.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.426899 15083 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
14:01:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.430282 15097 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.431948 15083 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
14:01:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.432098 15083 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.432839 15083 fs_report.cc:389] FS layout report
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/data
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:23.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.446106 15083 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:23.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.446936 15083 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:23.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.447192 15083 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:23.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.448194 15083 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.450537 15104 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:23.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.457679 15083 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:23.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.457803 15083 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
14:01:23.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.457921 15083 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:23.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.462458 15083 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:23.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.462576 15083 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.000s	sys 0.002s
14:01:23.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.463168 15104 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap starting.
14:01:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.490557 15104 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Log is configured to *not* fsync() on all Append() calls
14:01:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.506631 15083 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.65:44239
14:01:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.506695 15211 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.65:44239 every 8 connection(s)
14:01:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.507088 15104 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.507742 15104 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap complete.
14:01:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.508190 15083 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.509862 15104 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent bootstrapping tablet: real 0.047s	user 0.032s	sys 0.011s
14:01:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.512527 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 15083
14:01:23.513 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:23.531 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:23.532 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.66:46279
14:01:23.533 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:01:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.533332 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 14719
14:01:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.553273 15104 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.556686 15104 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Initialized, Role: FOLLOWER
14:01:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.560286 15104 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.567678 15104 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent starting tablet: real 0.058s	user 0.009s	sys 0.003s
14:01:23.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.570402 15213 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.570930 15213 heartbeater.cc:461] Registering TS with master...
14:01:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.572691 15213 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.576511 14343 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:23.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.580642 14841 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.582286 15214 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.582448 15214 heartbeater.cc:461] Registering TS with master...
14:01:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.582911 15214 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.583566 15212 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.583747 15212 heartbeater.cc:461] Registering TS with master...
14:01:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.584334 15212 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:23.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.586153 14484 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:23.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.587795 14484 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.65:38231
14:01:23.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.588393 14413 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.590037 15212 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:23.685935 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 14719
14:01:23.704 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:23.705 [DEBUG - kudu-nio-2] (Connection.java:239) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] handling channelInactive
14:01:23.705 [DEBUG - kudu-nio-2] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state READY due to: connection closed
14:01:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:23.706461 15226 negotiation.cc:336] Failed RPC negotiation. Trace:
14:01:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:23.685319 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.13.247.66:45411 (local address 127.13.247.65:44239)
14:01:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:23.685944 (+   625us) server_negotiation.cc:207] Beginning negotiation
14:01:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:23.685955 (+    11us) server_negotiation.cc:400] Waiting for connection header
14:01:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:23.704939 (+ 18984us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.247.66:45411: BlockingRecv error: recv got EOF from 127.13.247.66:45411 (error 108)
14:01:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":449,"thread_start_us":499,"threads_started":1}
14:01:23.715 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:23.716 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.716 [DEBUG - kudu-nio-3] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.717 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.737 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:23.738 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:23.738 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=3, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [11ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [32ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1060106956(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1138e9c9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d75552b -> wakeup thread Time-limited test)) after lookup
14:01:23.740 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:23.741 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.741 [DEBUG - kudu-nio-4] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.741 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.777 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:23.778 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:23.778 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=5, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [11ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [32ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [34ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [36ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [36ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [72ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1060106956(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1138e9c9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d75552b -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216))) after lookup
14:01:23.779 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:23.780 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.780 [DEBUG - kudu-nio-5] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.781 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.817 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:23.817 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:23.817 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=7, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=112), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [11ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [32ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [34ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [36ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [36ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [72ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [74ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [76ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [76ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [111ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1060106956(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1138e9c9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d75552b -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183))) after lookup
14:01:23.818 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:23.819 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.819 [DEBUG - kudu-nio-6] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:23.820 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:24.097 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:24.098 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:24.098 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=9, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=393), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [11ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [32ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [34ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [36ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [36ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [72ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [74ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [76ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [76ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [111ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [113ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [115ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [115ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [388ms] refreshing cache from master, [388ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [392ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1060106956(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1138e9c9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183) -> (continuation of Deferred@727455509 after retry RPC@72511261), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d75552b -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183) -> (continuation of Deferred@727455509 after retry RPC@72511261))) after lookup
14:01:24.099 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:24.101 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:24.101 [DEBUG - kudu-nio-7] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:24.101 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:24.755476 15227 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:01:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:24.755739 15227 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:24.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:24.757372 15227 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:24.759418 15099 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:24.761024 15099 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:24.764878 14936 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082" is_pre_election: true
14:01:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:24.765785 15101 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99
14:01:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:24.766266 15227 raft_consensus.cc:2749] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:01:24.778 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:24.778 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:24.778 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=11, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1073), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [11ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [32ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [34ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [36ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [36ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [72ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [74ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [76ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [76ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [111ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [113ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [115ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [115ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [388ms] refreshing cache from master, [388ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [392ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [394ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [396ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [396ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1069ms] refreshing cache from master, [1069ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1072ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1060106956(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1138e9c9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183) -> (continuation of Deferred@727455509 after retry RPC@72511261) -> (continuation of Deferred@419362444 after retry RPC@398846513), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d75552b -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183) -> (continuation of Deferred@727455509 after retry RPC@72511261) -> (continuation of Deferred@419362444 after retry RPC@398846513))) after lookup
14:01:24.779 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:24.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:24.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:24.780 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.065917 15232 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader bde3d574cb824a4aa08d2b3f34754f99)
14:01:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.066185 15232 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.067160 15232 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:25.071110 14870 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.075932 15166 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "fafc4088d51349e7973eb5b714841aa8" is_pre_election: true
14:01:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.076380 15166 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 1.
14:01:25.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.077450 14870 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8; no voters: bde3d574cb824a4aa08d2b3f34754f99
14:01:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.077809 15232 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:01:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.077976 15232 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Starting leader election (detected failure of leader bde3d574cb824a4aa08d2b3f34754f99)
14:01:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.078089 15232 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 2 FOLLOWER]: Advancing to term 3
14:01:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.080246 15232 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:25.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.080889 15232 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 3 election: Requested vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:25.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.081488 15166 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:25.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.081703 15166 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 1 FOLLOWER]: Advancing to term 3
14:01:25.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:25.082741 14870 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:25.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.085877 15166 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 3.
14:01:25.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.086455 14870 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8; no voters: bde3d574cb824a4aa08d2b3f34754f99
14:01:25.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.086766 15232 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 3 FOLLOWER]: Leader election won for term 3
14:01:25.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.087256 15232 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 3 LEADER]: Becoming Leader. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Running, Role: LEADER
14:01:25.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.087833 15232 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.091465 14484 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 reported cstate change: term changed from 2 to 3, leader changed from bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66) to 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67). New cstate: current_term: 3 leader_uuid: "27ce0884bf764cdbb025c6d3b9edc082" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.522902 15166 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 3 FOLLOWER]: Refusing update from remote peer 27ce0884bf764cdbb025c6d3b9edc082: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
14:01:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:25.523792 15232 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
14:01:25.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:25.540632 14870 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 -> Peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Couldn't send request to peer bde3d574cb824a4aa08d2b3f34754f99. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:26.877 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:26.878 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:26.878 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=13, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=3173), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [11ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [32ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [34ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [36ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [36ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [72ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [74ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [76ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [76ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [111ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [113ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [115ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [115ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [388ms] refreshing cache from master, [388ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [392ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [394ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [396ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [396ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1069ms] refreshing cache from master, [1069ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1072ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [1073ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1075ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1075ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [3168ms] refreshing cache from master, [3168ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [3172ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1060106956(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1138e9c9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183) -> (continuation of Deferred@727455509 after retry RPC@72511261) -> (continuation of Deferred@419362444 after retry RPC@398846513) -> (continuation of Deferred@1017916264 after retry RPC@1062573135), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1d75552b -> wakeup thread Time-limited test -> (continuation of Deferred@1507381845 after retry RPC@1848838216) -> (continuation of Deferred@2013467327 after retry RPC@243541183) -> (continuation of Deferred@727455509 after retry RPC@72511261) -> (continuation of Deferred@419362444 after retry RPC@398846513) -> (continuation of Deferred@1017916264 after retry RPC@1062573135))) after lookup
14:01:26.879 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:26.881 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] Successfully connected to peer
14:01:26.899 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xfe795a2d, L:/127.0.0.1:36902 - R:/127.13.247.67:36071] using TOKEN/null
14:01:26.929 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.66:46279
14:01:26.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:01:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:26.931653 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:46279
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36973
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.149892 14870 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111) [suppressed 5 similar messages]
14:01:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.288683 15270 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.289129 15270 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.289269 15270 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.298206 15270 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.298439 15270 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.66
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.308877 15270 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:46279
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36973
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.310662 15270 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.312659 15270 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.322252 15278 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.322306 15277 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.323172 15280 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:27.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.324196 15270 server_base.cc:1061] running on GCE node
14:01:27.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.324911 15270 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:27.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.326313 15270 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:27.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.327572 15270 hybrid_clock.cc:648] HybridClock initialized: now 1783519287327512 us; error 44 us; skew 500 ppm
14:01:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.327955 15270 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.330435 15270 webserver.cc:533] Webserver started at http://127.13.247.66:36973/ using document root <none> and password file <none>
14:01:27.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.331353 15270 fs_manager.cc:362] Metadata directory not provided
14:01:27.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.331521 15270 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.337792 15270 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
14:01:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.341341 15286 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.343180 15270 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.000s
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.343429 15270 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.344214 15270 fs_report.cc:389] FS layout report
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/data
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.360846 15270 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.361783 15270 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.362072 15270 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.363041 15270 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.365373 15293 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:27.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.371503 15270 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:27.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.371657 15270 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.000s	sys 0.001s
14:01:27.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.371809 15270 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:27.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.375476 15270 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:27.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.375643 15270 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.001s
14:01:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.376201 15293 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap starting.
14:01:27.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.404511 15293 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Log is configured to *not* fsync() on all Append() calls
14:01:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.425684 15270 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.66:46279
14:01:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.425732 15400 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.66:46279 every 8 connection(s)
14:01:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.427681 15270 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.431970 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 15270
14:01:27.432 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.439716 15293 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:27.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.440726 15293 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap complete.
14:01:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.442935 15293 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent bootstrapping tablet: real 0.067s	user 0.042s	sys 0.017s
14:01:27.446 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:27.447 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.67:36071
14:01:27.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:01:27.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.448308 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 14853
14:01:27.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.456699 15293 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.457772 15293 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: bde3d574cb824a4aa08d2b3f34754f99, State: Initialized, Role: FOLLOWER
14:01:27.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.458639 15293 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.468742 15293 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent starting tablet: real 0.026s	user 0.017s	sys 0.005s
14:01:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.479720 14976 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.485852 15402 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.485927 15401 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.486340 15402 heartbeater.cc:461] Registering TS with master...
14:01:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.486373 15401 heartbeater.cc:461] Registering TS with master...
14:01:27.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.487641 15402 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:27.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.487663 15401 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:27.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.492066 15403 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:27.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.492300 14342 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:27.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.492605 15403 heartbeater.cc:461] Registering TS with master...
14:01:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.493263 14485 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.494897 14485 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.66:40115
14:01:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.495713 15403 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.497349 14413 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.498721 15401 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:27.617507 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 14853
14:01:27.637 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] handling channelInactive
14:01:27.637 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state READY due to: connection closed
14:01:27.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.637449 15416 negotiation.cc:336] Failed RPC negotiation. Trace:
14:01:27.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.615596 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.13.247.67:38157 (local address 127.13.247.66:46279)
14:01:27.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.616202 (+   606us) server_negotiation.cc:207] Beginning negotiation
14:01:27.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.616211 (+     9us) server_negotiation.cc:400] Waiting for connection header
14:01:27.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.616248 (+    37us) server_negotiation.cc:408] Connection header received
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.616359 (+   111us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.616368 (+     9us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.616687 (+   319us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.616943 (+   256us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.618471 (+  1528us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.619943 (+  1472us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:27.636244 (+ 16301us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.247.67:38157: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":445,"thread_start_us":732,"threads_started":1}
14:01:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:27.638209 14471 connection.cc:573] server connection from 127.13.247.67:60669 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:27.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:27.639 [DEBUG - Time-limited test] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:27.640 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.640 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.640 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:27.658 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:27.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [18ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@785918644(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@78999a94 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2def1888 -> wakeup thread Time-limited test)) after lookup
14:01:27.659 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:27.660 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.660 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.660 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:27.677 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:27.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [18ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [20ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [21ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [22ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [37ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@785918644(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@78999a94 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2def1888 -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299))) after lookup
14:01:27.678 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:27.679 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.679 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.679 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:27.717 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:27.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [18ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [20ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [21ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [22ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [37ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [39ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [40ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [40ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [77ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@785918644(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@78999a94 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2def1888 -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695))) after lookup
14:01:27.718 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:27.719 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.719 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.719 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:27.837 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:27.837 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=198), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [18ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [20ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [21ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [22ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [37ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [39ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [40ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [40ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [77ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [79ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [80ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [80ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [194ms] refreshing cache from master, [194ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [197ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@785918644(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@78999a94 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695) -> (continuation of Deferred@655552069 after retry RPC@924211294), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2def1888 -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695) -> (continuation of Deferred@655552069 after retry RPC@924211294))) after lookup
14:01:27.838 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:27.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.840 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:27.840 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:28.017 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:28.017 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:28.018 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=379), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [18ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [20ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [21ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [22ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [37ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [39ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [40ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [40ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [77ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [79ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [80ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [80ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [194ms] refreshing cache from master, [194ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [197ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [199ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [201ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [201ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [374ms] refreshing cache from master, [375ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [378ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@785918644(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@78999a94 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695) -> (continuation of Deferred@655552069 after retry RPC@924211294) -> (continuation of Deferred@369440178 after retry RPC@332680068), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2def1888 -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695) -> (continuation of Deferred@655552069 after retry RPC@924211294) -> (continuation of Deferred@369440178 after retry RPC@332680068))) after lookup
14:01:28.018 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:28.020 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:28.020 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:28.020 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.884335 15417 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 27ce0884bf764cdbb025c6d3b9edc082)
14:01:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.884596 15417 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:28.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.885452 15417 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:28.887961 15101 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.893548 15355 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99" is_pre_election: true
14:01:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.894001 15355 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 2.
14:01:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.895004 15099 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082
14:01:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.895350 15417 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 3 FOLLOWER]: Leader pre-election won for term 4
14:01:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.895488 15417 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 27ce0884bf764cdbb025c6d3b9edc082)
14:01:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.895586 15417 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 3 FOLLOWER]: Advancing to term 4
14:01:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.897680 15417 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.898483 15417 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 4 election: Requested vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.899309 15355 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.899590 15355 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 2 FOLLOWER]: Advancing to term 4
14:01:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:28.900285 15101 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.904124 15355 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 4.
14:01:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.904778 15099 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082
14:01:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.905228 15417 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 4 FOLLOWER]: Leader election won for term 4
14:01:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.905658 15417 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 4 LEADER]: Becoming Leader. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Running, Role: LEADER
14:01:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.906261 15417 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:28.910472 14485 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 reported cstate change: term changed from 3 to 4, leader changed from 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67) to fafc4088d51349e7973eb5b714841aa8 (127.13.247.65). New cstate: current_term: 4 leader_uuid: "fafc4088d51349e7973eb5b714841aa8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:29.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.366641 15355 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 4 FOLLOWER]: Refusing update from remote peer fafc4088d51349e7973eb5b714841aa8: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
14:01:29.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.367702 15417 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
14:01:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.384399 15101 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 -> Peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Couldn't send request to peer 27ce0884bf764cdbb025c6d3b9edc082. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:29.476 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:29.477 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:29.477 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1838), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [18ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [20ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [21ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [22ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [37ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [39ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [40ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [40ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [77ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [79ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [80ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [80ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [194ms] refreshing cache from master, [194ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [197ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [199ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [201ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [201ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [374ms] refreshing cache from master, [375ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [378ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [379ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [381ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [381ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1834ms] refreshing cache from master, [1834ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1837ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@785918644(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@78999a94 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695) -> (continuation of Deferred@655552069 after retry RPC@924211294) -> (continuation of Deferred@369440178 after retry RPC@332680068) -> (continuation of Deferred@1919499101 after retry RPC@1129371391), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2def1888 -> wakeup thread Time-limited test -> (continuation of Deferred@715251663 after retry RPC@2145364299) -> (continuation of Deferred@611434107 after retry RPC@644271695) -> (continuation of Deferred@655552069 after retry RPC@924211294) -> (continuation of Deferred@369440178 after retry RPC@332680068) -> (continuation of Deferred@1919499101 after retry RPC@1129371391))) after lookup
14:01:29.478 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:29.479 [DEBUG - kudu-nio-0] (Connection.java:757) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] Successfully connected to peer
14:01:29.494 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xb415b2ee, L:/127.0.0.1:49420 - R:/127.13.247.65:44239] using TOKEN/null
14:01:29.519 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.67:36071
14:01:29.519 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:01:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.520567 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:36071
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43327
14:01:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:29.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.851743 15101 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111) [suppressed 3 similar messages]
14:01:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.861691 15444 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.862075 15444 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.862205 15444 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:29.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.870968 15444 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:29.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.871235 15444 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.67
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.881728 15444 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:36071
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43327
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.884176 15444 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:29.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.886097 15444 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:29.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.896528 15449 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.896665 15450 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:29.897928 15452 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.898568 15444 server_base.cc:1061] running on GCE node
14:01:29.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.899408 15444 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.900835 15444 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.902119 15444 hybrid_clock.cc:648] HybridClock initialized: now 1783519289902060 us; error 49 us; skew 500 ppm
14:01:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.902452 15444 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.905022 15444 webserver.cc:533] Webserver started at http://127.13.247.67:43327/ using document root <none> and password file <none>
14:01:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.905819 15444 fs_manager.cc:362] Metadata directory not provided
14:01:29.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.905929 15444 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:29.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.912432 15444 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
14:01:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.915669 15458 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.917248 15444 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.917431 15444 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:18 on dist-test-slave-f6s0"
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.918200 15444 fs_report.cc:389] FS layout report
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/data
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.936380 15444 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.937373 15444 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.937711 15444 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.938751 15444 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:29.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.941215 15465 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.947918 15444 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.948029 15444 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:01:29.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.948138 15444 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:29.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.952217 15444 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:29.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.952333 15444 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.004s	sys 0.000s
14:01:29.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.952973 15465 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap starting.
14:01:29.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.977492 15465 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Log is configured to *not* fsync() on all Append() calls
14:01:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.996840 15444 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.67:36071
14:01:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.996891 15572 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.67:36071 every 8 connection(s)
14:01:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.998329 15444 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:29.999302 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 15444
14:01:29.999 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:30.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.000509 15465 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.001163 15465 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap complete.
14:01:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.009157 15465 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent bootstrapping tablet: real 0.057s	user 0.028s	sys 0.020s
14:01:30.034 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:30.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.036762 15465 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:30.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.037803 15465 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Initialized, Role: FOLLOWER
14:01:30.041 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.65:44239
14:01:30.042 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:30.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.042440 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 15083
14:01:30.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.048098 15465 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.051308 15573 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.051813 15573 heartbeater.cc:461] Registering TS with master...
14:01:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.053161 15573 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:30.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.055189 15577 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:30.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.055369 15577 heartbeater.cc:461] Registering TS with master...
14:01:30.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.055958 15577 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:30.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.057281 14484 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:30.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.057507 14413 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:30.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.059160 14484 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.67:51307
14:01:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.061401 15574 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.061568 15574 heartbeater.cc:461] Registering TS with master...
14:01:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.061882 15574 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.063367 14342 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:30.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.072458 15465 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent starting tablet: real 0.063s	user 0.013s	sys 0.002s
14:01:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.073370 15573 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:30.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.078348 15207 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:30.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:30.176393 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 15083
14:01:30.195 [DEBUG - kudu-nio-0] (Connection.java:239) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] handling channelInactive
14:01:30.195 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:30.196 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state READY due to: connection closed
14:01:30.196 [DEBUG - Time-limited test] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:30.197 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.198 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.216 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:30.216 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:30.217 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1108833810(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@3df769f9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a31d94d -> wakeup thread Time-limited test)) after lookup
14:01:30.217 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:30.218 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.219 [DEBUG - kudu-nio-2] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.219 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:30.237 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:30.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [41ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1108833810(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@3df769f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a31d94d -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258))) after lookup
14:01:30.238 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:30.240 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.240 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.240 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:30.257 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:30.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [41ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [42ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [44ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [44ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1108833810(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@3df769f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a31d94d -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598))) after lookup
14:01:30.258 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:30.259 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.259 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.259 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.357 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:30.357 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:30.358 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=162), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [41ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [42ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [44ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [44ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [161ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1108833810(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@3df769f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a31d94d -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245))) after lookup
14:01:30.359 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:30.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.361 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:30.397 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:30.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=201), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [41ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [42ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [44ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [44ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [161ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [163ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [165ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [165ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [197ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [200ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1108833810(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@3df769f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245) -> (continuation of Deferred@236744513 after retry RPC@1474983323), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a31d94d -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245) -> (continuation of Deferred@236744513 after retry RPC@1474983323))) after lookup
14:01:30.398 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:30.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.400 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.497 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:30.497 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:30.497 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=301), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [41ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [42ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [44ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [44ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [161ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [163ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [165ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [165ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [197ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [200ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [202ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [204ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [204ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [297ms] refreshing cache from master, [298ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [300ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1108833810(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@3df769f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245) -> (continuation of Deferred@236744513 after retry RPC@1474983323) -> (continuation of Deferred@1749406720 after retry RPC@1812410942), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a31d94d -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245) -> (continuation of Deferred@236744513 after retry RPC@1474983323) -> (continuation of Deferred@1749406720 after retry RPC@1812410942))) after lookup
14:01:30.498 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:30.499 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.499 [DEBUG - kudu-nio-7] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:30.500 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.401556 15588 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:31.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.401782 15588 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.403396 15588 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:31.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:31.405470 15288 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:31.407063 15288 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.412073 15528 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082" is_pre_election: true
14:01:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.412606 15528 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bde3d574cb824a4aa08d2b3f34754f99 in term 3.
14:01:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.413573 15290 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: fafc4088d51349e7973eb5b714841aa8
14:01:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.413982 15588 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 4 FOLLOWER]: Leader pre-election won for term 5
14:01:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.414111 15588 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 4 FOLLOWER]: Starting leader election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.414219 15588 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 4 FOLLOWER]: Advancing to term 5
14:01:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.416496 15588 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.417172 15588 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 5 election: Requested vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.417829 15528 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.418051 15528 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 3 FOLLOWER]: Advancing to term 5
14:01:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:31.418684 15288 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.423159 15528 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bde3d574cb824a4aa08d2b3f34754f99 in term 5.
14:01:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.423834 15290 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: fafc4088d51349e7973eb5b714841aa8
14:01:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.424320 15588 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 5 FOLLOWER]: Leader election won for term 5
14:01:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.424795 15588 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 5 LEADER]: Becoming Leader. State: Replica: bde3d574cb824a4aa08d2b3f34754f99, State: Running, Role: LEADER
14:01:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.425726 15588 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.429957 14484 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 reported cstate change: term changed from 4 to 5, leader changed from fafc4088d51349e7973eb5b714841aa8 (127.13.247.65) to bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66). New cstate: current_term: 5 leader_uuid: "bde3d574cb824a4aa08d2b3f34754f99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: HEALTHY } } }
14:01:31.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.843554 15528 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 5 FOLLOWER]: Refusing update from remote peer bde3d574cb824a4aa08d2b3f34754f99: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
14:01:31.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:31.844398 15288 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 -> Peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Couldn't send request to peer fafc4088d51349e7973eb5b714841aa8. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:31.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:31.844645 15588 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
14:01:33.958 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:33.959 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:33.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=3763), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [41ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [42ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [44ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [44ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [161ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [163ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [165ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [165ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [197ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [200ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [202ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [204ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [204ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [297ms] refreshing cache from master, [298ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [300ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [302ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [303ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [304ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [3758ms] refreshing cache from master, [3758ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [3762ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1108833810(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@3df769f9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245) -> (continuation of Deferred@236744513 after retry RPC@1474983323) -> (continuation of Deferred@1749406720 after retry RPC@1812410942) -> (continuation of Deferred@738940988 after retry RPC@2043152705), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1a31d94d -> wakeup thread Time-limited test -> (continuation of Deferred@1450380136 after retry RPC@1117543258) -> (continuation of Deferred@126434206 after retry RPC@670033598) -> (continuation of Deferred@2001473148 after retry RPC@793693245) -> (continuation of Deferred@236744513 after retry RPC@1474983323) -> (continuation of Deferred@1749406720 after retry RPC@1812410942) -> (continuation of Deferred@738940988 after retry RPC@2043152705))) after lookup
14:01:33.961 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:33.963 [DEBUG - kudu-nio-0] (Connection.java:757) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] Successfully connected to peer
14:01:33.984 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x92ad3b90, L:/127.0.0.1:48416 - R:/127.13.247.66:46279] using TOKEN/null
14:01:34.017 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.65:44239
14:01:34.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.018802 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:34.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.371882 15629 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:34.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.372208 15629 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:34.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.372292 15629 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:34.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.381136 15629 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:34.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.381340 15629 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.65
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.392107 15629 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:34.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.394093 15629 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:34.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.396274 15629 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.406751 15636 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.407014 15637 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.407380 15639 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.408397 15629 server_base.cc:1061] running on GCE node
14:01:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.409104 15629 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.410249 15629 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.411466 15629 hybrid_clock.cc:648] HybridClock initialized: now 1783519294411392 us; error 67 us; skew 500 ppm
14:01:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.411805 15629 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:34.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.414436 15629 webserver.cc:533] Webserver started at http://127.13.247.65:32823/ using document root <none> and password file <none>
14:01:34.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.415302 15629 fs_manager.cc:362] Metadata directory not provided
14:01:34.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.415426 15629 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.421614 15629 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.002s
14:01:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.425179 15645 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:34.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.427098 15629 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.427321 15629 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.428328 15629 fs_report.cc:389] FS layout report
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/data
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.466666 15629 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.467576 15629 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.467840 15629 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.468847 15629 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.471131 15652 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.477623 15629 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.477737 15629 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:01:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.477842 15629 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.481262 15629 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.481453 15629 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.002s
14:01:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.482178 15652 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap starting.
14:01:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.510445 15652 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Log is configured to *not* fsync() on all Append() calls
14:01:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.525596 15629 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.65:44239
14:01:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.525627 15759 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.65:44239 every 8 connection(s)
14:01:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.527782 15629 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.530984 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 15629
14:01:34.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:34.554 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:34.560 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.66:46279
14:01:34.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:01:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.560878 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 15270
14:01:34.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.569600 15761 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:34.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.569646 15760 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:34.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.570003 15761 heartbeater.cc:461] Registering TS with master...
14:01:34.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.570009 15760 heartbeater.cc:461] Registering TS with master...
14:01:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.571434 15761 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.571476 15760 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.575140 14484 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.576736 14484 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.65:47139
14:01:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.582547 14343 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:34.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.590634 15652 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:34.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.591591 15652 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap complete.
14:01:34.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.593398 15652 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent bootstrapping tablet: real 0.112s	user 0.047s	sys 0.018s
14:01:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.598706 15652 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.599763 15652 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Initialized, Role: FOLLOWER
14:01:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.600199 15762 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.600430 15762 heartbeater.cc:461] Registering TS with master...
14:01:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.600643 15652 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.600889 15762 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.602433 14414 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.602545 15396 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.602620 15652 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent starting tablet: real 0.009s	user 0.008s	sys 0.000s
14:01:34.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.604306 15760 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:34.713712 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 15270
14:01:34.739 [DEBUG - kudu-nio-0] (Connection.java:239) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] handling channelInactive
14:01:34.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state READY due to: connection closed
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:34.740104 15775 negotiation.cc:336] Failed RPC negotiation. Trace:
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.704605 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.13.247.66:60297 (local address 127.13.247.65:44239)
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.713526 (+  8921us) server_negotiation.cc:207] Beginning negotiation
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.713546 (+    20us) server_negotiation.cc:400] Waiting for connection header
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.713591 (+    45us) server_negotiation.cc:408] Connection header received
14:01:34.740 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.713708 (+   117us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.713714 (+     6us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.714702 (+   988us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.714944 (+   242us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:34.738336 (+ 23392us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.247.66:60297: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":8743,"thread_start_us":8492,"threads_started":1}
14:01:34.741 [DEBUG - Time-limited test] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:34.742 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.742 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.742 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.757 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:34.757 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:34.757 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1187680360(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@57827db4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2378202b -> wakeup thread Time-limited test)) after lookup
14:01:34.758 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:34.759 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.759 [DEBUG - kudu-nio-2] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.759 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.776 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:34.776 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:34.776 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [16ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [18ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [18ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [35ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1187680360(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@57827db4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2378202b -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368))) after lookup
14:01:34.777 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:34.779 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.779 [DEBUG - kudu-nio-3] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.779 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.817 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:34.817 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:34.818 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [16ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [18ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [18ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [35ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [38ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [38ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [76ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1187680360(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@57827db4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2378202b -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098))) after lookup
14:01:34.818 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:34.819 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.819 [DEBUG - kudu-nio-4] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.820 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.916 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:34.916 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:34.917 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=176), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [16ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [18ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [18ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [35ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [38ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [38ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [76ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [77ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [79ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [79ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [175ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1187680360(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@57827db4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2378202b -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390))) after lookup
14:01:34.918 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:34.919 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.919 [DEBUG - kudu-nio-5] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:34.919 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:35.337 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:35.338 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:35.338 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=597), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [16ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [18ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [18ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [35ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [38ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [38ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [76ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [77ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [79ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [79ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [175ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [176ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [178ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [178ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [593ms] refreshing cache from master, [593ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [596ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1187680360(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@57827db4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390) -> (continuation of Deferred@1125177531 after retry RPC@407841760), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2378202b -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390) -> (continuation of Deferred@1125177531 after retry RPC@407841760))) after lookup
14:01:35.339 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:35.340 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:35.340 [DEBUG - kudu-nio-6] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:35.340 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:35.536 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:35.537 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:35.537 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=796), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [16ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [18ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [18ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [35ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [38ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [38ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [76ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [77ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [79ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [79ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [175ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [176ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [178ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [178ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [593ms] refreshing cache from master, [593ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [596ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [597ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [599ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [599ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [792ms] refreshing cache from master, [792ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [795ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1187680360(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@57827db4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390) -> (continuation of Deferred@1125177531 after retry RPC@407841760) -> (continuation of Deferred@417474556 after retry RPC@2039680811), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2378202b -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390) -> (continuation of Deferred@1125177531 after retry RPC@407841760) -> (continuation of Deferred@417474556 after retry RPC@2039680811))) after lookup
14:01:35.538 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:35.539 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:35.539 [DEBUG - kudu-nio-7] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:35.539 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:36.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.094692 15776 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader bde3d574cb824a4aa08d2b3f34754f99)
14:01:36.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.095063 15776 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.097028 15776 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:36.098924 15460 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:36.100613 15460 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.105031 15714 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "fafc4088d51349e7973eb5b714841aa8" is_pre_election: true
14:01:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.105576 15714 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 4.
14:01:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.106657 15460 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8; no voters: bde3d574cb824a4aa08d2b3f34754f99
14:01:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.107137 15776 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 5 FOLLOWER]: Leader pre-election won for term 6
14:01:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.107321 15776 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 5 FOLLOWER]: Starting leader election (detected failure of leader bde3d574cb824a4aa08d2b3f34754f99)
14:01:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.107402 15776 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 5 FOLLOWER]: Advancing to term 6
14:01:36.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.109990 15776 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:36.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.110572 15776 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 6 election: Requested vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:36.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.111172 15714 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:36.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.111449 15714 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 4 FOLLOWER]: Advancing to term 6
14:01:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:36.112293 15460 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.116061 15714 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 6.
14:01:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.116762 15460 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8; no voters: bde3d574cb824a4aa08d2b3f34754f99
14:01:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.117307 15776 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Leader election won for term 6
14:01:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.117815 15776 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 LEADER]: Becoming Leader. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Running, Role: LEADER
14:01:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.118423 15776 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.122427 14484 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 reported cstate change: term changed from 5 to 6, leader changed from bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66) to 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67). New cstate: current_term: 6 leader_uuid: "27ce0884bf764cdbb025c6d3b9edc082" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:36.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:36.537701 15460 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 -> Peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Couldn't send request to peer bde3d574cb824a4aa08d2b3f34754f99. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:36.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.638126 15714 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 6 FOLLOWER]: Refusing update from remote peer 27ce0884bf764cdbb025c6d3b9edc082: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
14:01:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:36.639143 15776 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
14:01:37.437 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:37.437 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:37.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2697), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [1ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [1ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [16ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [18ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [18ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [35ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [38ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [38ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [76ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [77ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [79ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [79ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [172ms] refreshing cache from master, [172ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [175ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [176ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [178ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [178ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [593ms] refreshing cache from master, [593ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [596ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [597ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [599ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [599ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [792ms] refreshing cache from master, [792ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [795ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [797ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [798ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [798ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2692ms] refreshing cache from master, [2693ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [2696ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1187680360(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@57827db4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390) -> (continuation of Deferred@1125177531 after retry RPC@407841760) -> (continuation of Deferred@417474556 after retry RPC@2039680811) -> (continuation of Deferred@1183880337 after retry RPC@656089407), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2378202b -> wakeup thread Time-limited test -> (continuation of Deferred@1295449229 after retry RPC@799925368) -> (continuation of Deferred@465976366 after retry RPC@1527289098) -> (continuation of Deferred@314947665 after retry RPC@226152390) -> (continuation of Deferred@1125177531 after retry RPC@407841760) -> (continuation of Deferred@417474556 after retry RPC@2039680811) -> (continuation of Deferred@1183880337 after retry RPC@656089407))) after lookup
14:01:37.439 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:37.440 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] Successfully connected to peer
14:01:37.459 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x06f6d3ba, L:/127.0.0.1:60938 - R:/127.13.247.67:36071] using TOKEN/null
14:01:37.483 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.66:46279
14:01:37.485 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.486421 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:46279
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36973
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:37.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.840894 15808 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:37.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.841231 15808 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:37.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.841326 15808 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.850997 15808 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.851182 15808 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.66
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.862658 15808 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:46279
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36973
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.864439 15808 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.866307 15808 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.876334 15814 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.876343 15813 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:37.876538 15816 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.877839 15808 server_base.cc:1061] running on GCE node
14:01:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.878407 15808 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:37.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.879460 15808 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:37.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.880667 15808 hybrid_clock.cc:648] HybridClock initialized: now 1783519297880620 us; error 53 us; skew 500 ppm
14:01:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.880964 15808 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.883596 15808 webserver.cc:533] Webserver started at http://127.13.247.66:36973/ using document root <none> and password file <none>
14:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.884395 15808 fs_manager.cc:362] Metadata directory not provided
14:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.884490 15808 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:37.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.890136 15808 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.004s
14:01:37.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.893150 15822 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.894814 15808 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
14:01:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.894960 15808 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.895816 15808 fs_report.cc:389] FS layout report
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/data
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.940418 15808 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.941304 15808 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.941557 15808 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.942632 15808 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:37.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.944875 15829 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.951860 15808 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.951972 15808 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
14:01:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.952104 15808 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:37.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.955823 15808 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:37.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.955974 15808 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
14:01:37.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.956519 15829 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap starting.
14:01:37.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:37.992110 15829 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Log is configured to *not* fsync() on all Append() calls
14:01:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.008256 15808 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.66:46279
14:01:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.008315 15937 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.66:46279 every 8 connection(s)
14:01:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.009984 15808 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.011703 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 15808
14:01:38.012 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.043797 15829 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.044516 15829 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap complete.
14:01:38.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.046098 15829 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent bootstrapping tablet: real 0.090s	user 0.062s	sys 0.008s
14:01:38.074 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.073253 15829 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.074638 15829 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: bde3d574cb824a4aa08d2b3f34754f99, State: Initialized, Role: FOLLOWER
14:01:38.075 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.67:36071
14:01:38.075 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:01:38.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.075804 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 15444
14:01:38.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.075829 15829 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:38.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.078269 15829 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent starting tablet: real 0.032s	user 0.009s	sys 0.004s
14:01:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.087337 15938 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.087790 15938 heartbeater.cc:461] Registering TS with master...
14:01:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.088960 15938 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.092095 15939 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.092309 15939 heartbeater.cc:461] Registering TS with master...
14:01:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.092422 15945 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.092609 15945 heartbeater.cc:461] Registering TS with master...
14:01:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.093034 15939 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.093075 14485 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.093248 15945 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.094660 14343 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.095188 14485 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.66:50565
14:01:38.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.098423 15938 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:38.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.098883 14414 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:38.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.102563 15568 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:38.202445 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 15444
14:01:38.221 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] handling channelInactive
14:01:38.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state READY due to: connection closed
14:01:38.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:38.222 [DEBUG - Time-limited test] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:38.223 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.224 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:38.224103 15953 negotiation.cc:336] Failed RPC negotiation. Trace:
14:01:38.224 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.202181 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.13.247.67:54805 (local address 127.13.247.66:46279)
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.203350 (+  1169us) server_negotiation.cc:207] Beginning negotiation
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.203363 (+    13us) server_negotiation.cc:400] Waiting for connection header
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.203402 (+    39us) server_negotiation.cc:408] Connection header received
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.203529 (+   127us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.203540 (+    11us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.203856 (+   316us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.204086 (+   230us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:38.221121 (+ 17035us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.247.67:54805: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":958,"thread_start_us":328,"threads_started":1}
14:01:38.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:38.238 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:38.238 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test)) after lookup
14:01:38.239 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:38.240 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.240 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.241 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:38.257 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:38.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [17ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [19ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [19ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [34ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208))) after lookup
14:01:38.258 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:38.259 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.260 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.260 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.297 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:38.297 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:38.297 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [17ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [19ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [19ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [34ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [38ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [38ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [74ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612))) after lookup
14:01:38.298 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:38.299 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.299 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.299 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:38.397 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:38.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=175), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [17ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [19ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [19ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [34ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [38ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [38ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [74ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [76ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [77ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [77ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [174ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177))) after lookup
14:01:38.398 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:38.399 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.400 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.400 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.797 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:38.798 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:38.798 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=576), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [17ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [19ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [19ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [34ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [38ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [38ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [74ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [76ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [77ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [77ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [174ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [176ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [178ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [178ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [572ms] refreshing cache from master, [572ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [575ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375))) after lookup
14:01:38.799 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:38.800 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.800 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.800 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:38.857 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:38.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=635), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [17ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [19ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [19ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [34ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [38ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [38ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [74ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [76ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [77ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [77ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [174ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [176ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [178ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [178ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [572ms] refreshing cache from master, [572ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [575ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [577ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [578ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [578ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [631ms] refreshing cache from master, [632ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [635ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375) -> (continuation of Deferred@691569549 after retry RPC@355685351), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375) -> (continuation of Deferred@691569549 after retry RPC@355685351))) after lookup
14:01:38.858 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:38.859 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.860 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:38.860 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:39.356 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:39.357 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:39.357 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1135), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [17ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [19ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [19ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [34ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [38ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [38ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [74ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [76ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [77ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [77ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [174ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [176ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [178ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [178ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [572ms] refreshing cache from master, [572ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [575ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [577ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [578ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [578ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [631ms] refreshing cache from master, [632ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [635ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [636ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [638ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [638ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1131ms] refreshing cache from master, [1131ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1134ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375) -> (continuation of Deferred@691569549 after retry RPC@355685351) -> (continuation of Deferred@62127086 after retry RPC@210779156), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375) -> (continuation of Deferred@691569549 after retry RPC@355685351) -> (continuation of Deferred@62127086 after retry RPC@210779156))) after lookup
14:01:39.358 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:39.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:39.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:39.359 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.449254 15954 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 27ce0884bf764cdbb025c6d3b9edc082)
14:01:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.449455 15954 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.451195 15954 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:39.453982 15649 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:39.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:39.456306 15649 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.462111 15892 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99" is_pre_election: true
14:01:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.462704 15892 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 5.
14:01:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.463944 15647 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082
14:01:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.464352 15954 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 6 FOLLOWER]: Leader pre-election won for term 7
14:01:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.464502 15954 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 27ce0884bf764cdbb025c6d3b9edc082)
14:01:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.464650 15954 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 6 FOLLOWER]: Advancing to term 7
14:01:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.467166 15954 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.467926 15954 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 7 election: Requested vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.468519 15892 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.468766 15892 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 5 FOLLOWER]: Advancing to term 7
14:01:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:39.469798 15649 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.472990 15892 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 7.
14:01:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.473503 15647 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082
14:01:39.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.473933 15954 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Leader election won for term 7
14:01:39.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.474429 15954 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 LEADER]: Becoming Leader. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Running, Role: LEADER
14:01:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.475072 15954 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.479439 14485 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 reported cstate change: term changed from 6 to 7, leader changed from 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67) to fafc4088d51349e7973eb5b714841aa8 (127.13.247.65). New cstate: current_term: 7 leader_uuid: "fafc4088d51349e7973eb5b714841aa8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:39.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.890704 15892 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 7 FOLLOWER]: Refusing update from remote peer fafc4088d51349e7973eb5b714841aa8: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
14:01:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:39.891780 15954 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
14:01:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:39.906345 15649 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 -> Peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Couldn't send request to peer 27ce0884bf764cdbb025c6d3b9edc082. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:41.057 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:41.057 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:41.057 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=16, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2835), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [2ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [15ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [17ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [19ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [19ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [34ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [36ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [38ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [38ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [74ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [76ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [77ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [77ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [174ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [176ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [178ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [178ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [572ms] refreshing cache from master, [572ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [575ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [577ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [578ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [578ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [631ms] refreshing cache from master, [632ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [635ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [636ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [638ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [638ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1131ms] refreshing cache from master, [1131ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1134ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [1136ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1137ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1137ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [2831ms] refreshing cache from master, [2831ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [2834ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1512854509(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@2beb5e0e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375) -> (continuation of Deferred@691569549 after retry RPC@355685351) -> (continuation of Deferred@62127086 after retry RPC@210779156) -> (continuation of Deferred@2084916586 after retry RPC@108348236), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@589048cd -> wakeup thread Time-limited test -> (continuation of Deferred@942621683 after retry RPC@63660208) -> (continuation of Deferred@1640577829 after retry RPC@962529612) -> (continuation of Deferred@1524889328 after retry RPC@465015177) -> (continuation of Deferred@268335360 after retry RPC@685939375) -> (continuation of Deferred@691569549 after retry RPC@355685351) -> (continuation of Deferred@62127086 after retry RPC@210779156) -> (continuation of Deferred@2084916586 after retry RPC@108348236))) after lookup
14:01:41.058 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:41.059 [DEBUG - kudu-nio-1] (Connection.java:757) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] Successfully connected to peer
14:01:41.076 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x67a69c47, L:/127.0.0.1:60728 - R:/127.13.247.65:44239] using TOKEN/null
14:01:41.102 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.67:36071
14:01:41.102 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.103375 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:36071
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43327
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.444183 15991 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.444491 15991 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.444574 15991 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.453241 15991 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.453438 15991 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.67
14:01:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.465250 15991 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:36071
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43327
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.467121 15991 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:41.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.469141 15991 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.479627 15999 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.479753 15998 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.480548 15991 server_base.cc:1061] running on GCE node
14:01:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.480727 16001 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:41.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.481524 15991 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:41.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.482769 15991 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.484021 15991 hybrid_clock.cc:648] HybridClock initialized: now 1783519301483942 us; error 77 us; skew 500 ppm
14:01:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.484400 15991 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.486850 15991 webserver.cc:533] Webserver started at http://127.13.247.67:43327/ using document root <none> and password file <none>
14:01:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.487820 15991 fs_manager.cc:362] Metadata directory not provided
14:01:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.487958 15991 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.493587 15991 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
14:01:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.496546 16007 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.497956 15991 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
14:01:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.498090 15991 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:18 on dist-test-slave-f6s0"
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.498766 15991 fs_report.cc:389] FS layout report
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/data
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:41.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.524943 15991 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:41.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.525931 15991 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:41.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.526180 15991 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:41.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.529052 15991 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.537453 16014 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:41.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.544108 15991 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:41.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.544232 15991 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:01:41.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.544359 15991 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:41.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.547536 15991 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:41.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.547637 15991 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
14:01:41.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.548187 16014 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap starting.
14:01:41.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.574630 16014 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Log is configured to *not* fsync() on all Append() calls
14:01:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.596141 15991 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.67:36071
14:01:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.596160 16121 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.67:36071 every 8 connection(s)
14:01:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.597992 15991 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:41.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.602032 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 15991
14:01:41.603 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:41.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.617254 16014 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:41.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.618081 16014 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap complete.
14:01:41.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.619935 16014 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent bootstrapping tablet: real 0.072s	user 0.047s	sys 0.019s
14:01:41.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.629441 16014 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:41.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.630669 16014 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Initialized, Role: FOLLOWER
14:01:41.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.631585 16014 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.635732 16014 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent starting tablet: real 0.016s	user 0.016s	sys 0.001s
14:01:41.643 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:41.643 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.65:44239
14:01:41.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:41.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.644357 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 15629
14:01:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.652346 16124 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.652827 16124 heartbeater.cc:461] Registering TS with master...
14:01:41.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.654019 16124 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.655470 16122 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.655637 16122 heartbeater.cc:461] Registering TS with master...
14:01:41.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.656344 16122 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.657830 14414 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.658174 16123 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.658298 16123 heartbeater.cc:461] Registering TS with master...
14:01:41.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.658764 16123 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.664069 14484 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.664584 14343 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.665735 14484 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.67:59819
14:01:41.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.668707 16122 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.673409 15755 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:41.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:41.777287 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 15629
14:01:41.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.795037 15823 connection.cc:573] server connection from 127.13.247.65:33377 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:41.795 [DEBUG - kudu-nio-1] (Connection.java:239) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] handling channelInactive
14:01:41.796 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state READY due to: connection closed
14:01:41.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:41.796877 16138 negotiation.cc:336] Failed RPC negotiation. Trace:
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.776303 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.13.247.65:51257 (local address 127.13.247.67:36071)
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.776930 (+   627us) server_negotiation.cc:207] Beginning negotiation
14:01:41.797 [DEBUG - Time-limited test] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.776974 (+    44us) server_negotiation.cc:400] Waiting for connection header
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.777006 (+    32us) server_negotiation.cc:408] Connection header received
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.777139 (+   133us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.777147 (+     8us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.779343 (+  2196us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.779529 (+   186us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:41.795286 (+ 15757us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.247.65:51257: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":484,"thread_start_us":349,"threads_started":1}
14:01:41.798 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.798 [DEBUG - kudu-nio-2] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.798 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.817 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:41.817 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:41.817 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [2ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@851128764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1e786b84 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f922e40 -> wakeup thread Time-limited test)) after lookup
14:01:41.818 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:41.818 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.819 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.819 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:41.836 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:41.837 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [2ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [40ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@851128764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1e786b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f922e40 -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015))) after lookup
14:01:41.838 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:41.838 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.839 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.839 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:41.857 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:41.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [2ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [40ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [41ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [43ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [43ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@851128764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1e786b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f922e40 -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876))) after lookup
14:01:41.858 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:41.859 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.859 [DEBUG - kudu-nio-5] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.859 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.956 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:41.956 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:41.956 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=160), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [2ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [40ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [41ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [43ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [43ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [160ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@851128764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1e786b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f922e40 -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030))) after lookup
14:01:41.957 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:41.958 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.958 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:41.958 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:42.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:42.397 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:42.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=601), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [2ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [40ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [41ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [43ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [43ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [160ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [161ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [162ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [162ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [597ms] refreshing cache from master, [597ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [600ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@851128764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1e786b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030) -> (continuation of Deferred@517920661 after retry RPC@821267786), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f922e40 -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030) -> (continuation of Deferred@517920661 after retry RPC@821267786))) after lookup
14:01:42.398 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:42.399 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:42.399 [DEBUG - kudu-nio-7] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:42.399 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:42.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:42.858028 16139 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:01:42.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:42.858371 16139 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:42.860220 16139 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:42.861425 16009 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:42.863628 16009 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:42.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:42.867256 15892 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99" is_pre_election: true
14:01:42.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:42.868182 16009 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082; no voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8
14:01:42.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:42.868604 16139 raft_consensus.cc:2749] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
14:01:43.077 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:43.077 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:43.077 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1281), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [2ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [40ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [41ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [43ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [43ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [160ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [161ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [162ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [162ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [597ms] refreshing cache from master, [597ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [600ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [602ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [603ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [603ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [1278ms] refreshing cache from master, [1278ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1280ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@851128764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1e786b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030) -> (continuation of Deferred@517920661 after retry RPC@821267786) -> (continuation of Deferred@1992473111 after retry RPC@1970601543), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f922e40 -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030) -> (continuation of Deferred@517920661 after retry RPC@821267786) -> (continuation of Deferred@1992473111 after retry RPC@1970601543))) after lookup
14:01:43.078 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:43.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:43.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:43.080 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:43.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.278616 16144 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.278923 16144 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.280560 16144 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:43.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:43.281977 15824 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:43.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:43.284013 15824 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:43.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.289722 16077 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082" is_pre_election: true
14:01:43.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.290252 16077 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bde3d574cb824a4aa08d2b3f34754f99 in term 6.
14:01:43.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.291610 15826 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: fafc4088d51349e7973eb5b714841aa8
14:01:43.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.292093 16144 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 7 FOLLOWER]: Leader pre-election won for term 8
14:01:43.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.292258 16144 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 7 FOLLOWER]: Starting leader election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:43.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.292372 16144 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 7 FOLLOWER]: Advancing to term 8
14:01:43.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.294582 16144 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.295368 16144 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 8 election: Requested vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:43.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.296053 16077 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:43.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.296308 16077 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 6 FOLLOWER]: Advancing to term 8
14:01:43.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:43.297084 15824 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.300261 16077 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bde3d574cb824a4aa08d2b3f34754f99 in term 8.
14:01:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.300765 15826 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: fafc4088d51349e7973eb5b714841aa8
14:01:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.301195 16144 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Leader election won for term 8
14:01:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.301744 16144 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 LEADER]: Becoming Leader. State: Replica: bde3d574cb824a4aa08d2b3f34754f99, State: Running, Role: LEADER
14:01:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.302304 16144 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:43.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.306373 14484 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 reported cstate change: term changed from 7 to 8, leader changed from fafc4088d51349e7973eb5b714841aa8 (127.13.247.65) to bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66). New cstate: current_term: 8 leader_uuid: "bde3d574cb824a4aa08d2b3f34754f99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: HEALTHY } } }
14:01:43.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:43.695984 15824 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 -> Peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Couldn't send request to peer fafc4088d51349e7973eb5b714841aa8. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:43.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.799456 16077 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 8 FOLLOWER]: Refusing update from remote peer bde3d574cb824a4aa08d2b3f34754f99: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
14:01:43.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:43.800496 16144 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
14:01:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:45.865141 15824 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 -> Peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Couldn't send request to peer fafc4088d51349e7973eb5b714841aa8. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:01:45.877 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:45.877 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:45.878 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=4081), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [2ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [20ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [21ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [23ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [23ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [40ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [41ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [43ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [43ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [60ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [61ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [63ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [63ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [157ms] refreshing cache from master, [157ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [160ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [161ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [162ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [162ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [597ms] refreshing cache from master, [597ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [600ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [602ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [603ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [603ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [1278ms] refreshing cache from master, [1278ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1280ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [1282ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [1283ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [1284ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [4077ms] refreshing cache from master, [4077ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [4080ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@851128764(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@1e786b84 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030) -> (continuation of Deferred@517920661 after retry RPC@821267786) -> (continuation of Deferred@1992473111 after retry RPC@1970601543) -> (continuation of Deferred@391980550 after retry RPC@1872068182), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f922e40 -> wakeup thread Time-limited test -> (continuation of Deferred@368777134 after retry RPC@1540742015) -> (continuation of Deferred@304086131 after retry RPC@1734656876) -> (continuation of Deferred@1122729564 after retry RPC@55950030) -> (continuation of Deferred@517920661 after retry RPC@821267786) -> (continuation of Deferred@1992473111 after retry RPC@1970601543) -> (continuation of Deferred@391980550 after retry RPC@1872068182))) after lookup
14:01:45.878 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:45.880 [DEBUG - kudu-nio-1] (Connection.java:757) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] Successfully connected to peer
14:01:45.900 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xf251a176, L:/127.0.0.1:44020 - R:/127.13.247.66:46279] using TOKEN/null
14:01:45.926 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.65:44239
14:01:45.926 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:45.927414 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.269029 16182 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.269398 16182 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.269518 16182 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.278519 16182 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.278762 16182 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.65
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.289731 16182 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:46.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.292027 16182 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.294202 16182 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.304721 16188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.305366 16190 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:46.305546 16187 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.305888 16182 server_base.cc:1061] running on GCE node
14:01:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.306540 16182 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:46.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.307658 16182 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:46.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.315161 16182 hybrid_clock.cc:648] HybridClock initialized: now 1783519306315077 us; error 51 us; skew 500 ppm
14:01:46.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.315711 16182 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:46.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.318853 16182 webserver.cc:533] Webserver started at http://127.13.247.65:32823/ using document root <none> and password file <none>
14:01:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.319985 16182 fs_manager.cc:362] Metadata directory not provided
14:01:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.320138 16182 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.327966 16182 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.001s
14:01:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.331769 16196 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.333562 16182 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
14:01:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.333755 16182 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.334810 16182 fs_report.cc:389] FS layout report
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/data
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:46.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.379410 16182 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:46.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.380466 16182 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:46.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.380842 16182 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:46.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.382066 16182 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:46.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.384547 16203 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:46.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.393532 16182 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:46.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.393702 16182 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.001s
14:01:46.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.393850 16182 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.397233 16182 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.397382 16182 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.001s	sys 0.001s
14:01:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.398029 16203 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap starting.
14:01:46.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.425621 16203 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Log is configured to *not* fsync() on all Append() calls
14:01:46.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.463183 16203 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:46.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.463980 16203 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap complete.
14:01:46.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.465673 16203 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent bootstrapping tablet: real 0.068s	user 0.042s	sys 0.024s
14:01:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.474844 16203 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.476418 16203 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Initialized, Role: FOLLOWER
14:01:46.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.477407 16203 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:46.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.481568 16203 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent starting tablet: real 0.016s	user 0.012s	sys 0.004s
14:01:46.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.488627 16182 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.65:44239
14:01:46.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.488680 16314 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.65:44239 every 8 connection(s)
14:01:46.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.490298 16182 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:46.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.496380 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 16182
14:01:46.496 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:46.530 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:46.531 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.66:46279
14:01:46.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:01:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.531957 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 15808
14:01:46.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.553203 16315 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.553629 16315 heartbeater.cc:461] Registering TS with master...
14:01:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.555317 16316 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.555522 16316 heartbeater.cc:461] Registering TS with master...
14:01:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.555627 16315 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.556208 16316 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.558823 14343 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.558974 14485 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:46.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.560640 14485 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.65:51171
14:01:46.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.574244 15933 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:46.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.582154 16317 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:46.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.582379 16317 heartbeater.cc:461] Registering TS with master...
14:01:46.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.582870 16317 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:46.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.584815 14414 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:46.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:46.687269 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 15808
14:01:46.706 [DEBUG - kudu-nio-1] (Connection.java:239) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] handling channelInactive
14:01:46.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:46.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state READY due to: connection closed
14:01:46.706 [DEBUG - Time-limited test] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:46.707 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.707 [DEBUG - kudu-nio-2] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.708 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.716 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:46.716 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:46.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [2ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [10ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1311369178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@5dbe9fae -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@177ea6c8 -> wakeup thread Time-limited test)) after lookup
14:01:46.717 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:46.718 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.718 [DEBUG - kudu-nio-3] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.719 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.736 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:46.736 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:46.737 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [2ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [10ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [11ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [13ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [13ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1311369178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@5dbe9fae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@177ea6c8 -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027))) after lookup
14:01:46.737 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:46.738 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.738 [DEBUG - kudu-nio-4] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.738 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.776 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:46.776 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:46.776 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [2ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [10ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [11ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [13ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [13ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [32ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [70ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1311369178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@5dbe9fae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@177ea6c8 -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918))) after lookup
14:01:46.777 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:46.778 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.778 [DEBUG - kudu-nio-5] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.778 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:46.836 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:46.837 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=130), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [2ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [10ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [11ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [13ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [13ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [32ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [70ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [71ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [72ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [72ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [127ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [130ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1311369178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@5dbe9fae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918) -> (continuation of Deferred@647255930 after retry RPC@1515132223), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@177ea6c8 -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918) -> (continuation of Deferred@647255930 after retry RPC@1515132223))) after lookup
14:01:46.837 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:46.838 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.838 [DEBUG - kudu-nio-6] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:46.839 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:47.336 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:47.336 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)[L],fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:47.337 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=630), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [2ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [10ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [11ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [13ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [13ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [32ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [70ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [71ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [72ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [72ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [127ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [130ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [131ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [133ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [133ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [627ms] refreshing cache from master, [627ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [630ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1311369178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@5dbe9fae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918) -> (continuation of Deferred@647255930 after retry RPC@1515132223) -> (continuation of Deferred@1283262923 after retry RPC@1857624982), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@177ea6c8 -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918) -> (continuation of Deferred@647255930 after retry RPC@1515132223) -> (continuation of Deferred@1283262923 after retry RPC@1857624982))) after lookup
14:01:47.338 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] connecting to peer
14:01:47.339 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:47.339 [DEBUG - kudu-nio-7] (Connection.java:709) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] cleaning up while in state CONNECTING due to: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:47.339 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279
14:01:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:47.564345 16315 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:47.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:47.793524 16329 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:01:47.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:47.793818 16329 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:47.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:47.795538 16329 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:47.797179 16198 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:47.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:47.798751 16198 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:47.802896 16077 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082" is_pre_election: true
14:01:47.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:47.803874 16200 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99
14:01:47.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:47.804344 16329 raft_consensus.cc:2749] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
14:01:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.027477 16334 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader bde3d574cb824a4aa08d2b3f34754f99)
14:01:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.027729 16334 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.028571 16334 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:48.030305 16009 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:48.031932 16009 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:48.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.037511 16266 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "fafc4088d51349e7973eb5b714841aa8" is_pre_election: true
14:01:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.037868 16266 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 7.
14:01:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.038792 16009 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8; no voters: bde3d574cb824a4aa08d2b3f34754f99
14:01:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.039167 16334 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 8 FOLLOWER]: Leader pre-election won for term 9
14:01:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.039317 16334 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 8 FOLLOWER]: Starting leader election (detected failure of leader bde3d574cb824a4aa08d2b3f34754f99)
14:01:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.039388 16334 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 8 FOLLOWER]: Advancing to term 9
14:01:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.041462 16334 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.042146 16334 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 9 election: Requested vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.042806 16266 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.043063 16266 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 7 FOLLOWER]: Advancing to term 9
14:01:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:48.044087 16009 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111)
14:01:48.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.047071 16266 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 9.
14:01:48.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.047585 16009 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8; no voters: bde3d574cb824a4aa08d2b3f34754f99
14:01:48.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.047924 16334 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 9 FOLLOWER]: Leader election won for term 9
14:01:48.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.048489 16334 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 9 LEADER]: Becoming Leader. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Running, Role: LEADER
14:01:48.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.049146 16334 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:48.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.053148 14485 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 reported cstate change: term changed from 8 to 9, leader changed from bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66) to 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67). New cstate: current_term: 9 leader_uuid: "27ce0884bf764cdbb025c6d3b9edc082" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:48.465338 16009 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 -> Peer bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279): Couldn't send request to peer bde3d574cb824a4aa08d2b3f34754f99. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.66:46279: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:48.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.526449 16266 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 9 FOLLOWER]: Refusing update from remote peer 27ce0884bf764cdbb025c6d3b9edc082: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
14:01:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.527355 16334 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
14:01:48.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:48.836 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:48.837 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2130), Traces: [0ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [2ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [10ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [11ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [13ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [13ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [30ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [31ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [32ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [32ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [70ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [71ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [72ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [72ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [127ms] refreshing cache from master, [127ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [130ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [131ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [133ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [133ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [627ms] refreshing cache from master, [627ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [630ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [631ms] sending RPC to server bde3d574cb824a4aa08d2b3f34754f99, [633ms] received response from server bde3d574cb824a4aa08d2b3f34754f99: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [633ms] delaying RPC due to: Network error: Failed to connect to peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279): Connection refused: /127.13.247.66:46279, [2127ms] refreshing cache from master, [2127ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [2130ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1311369178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@5dbe9fae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918) -> (continuation of Deferred@647255930 after retry RPC@1515132223) -> (continuation of Deferred@1283262923 after retry RPC@1857624982) -> (continuation of Deferred@1706472671 after retry RPC@1140155988), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@177ea6c8 -> wakeup thread Time-limited test -> (continuation of Deferred@2078498825 after retry RPC@424264027) -> (continuation of Deferred@1236800003 after retry RPC@506220918) -> (continuation of Deferred@647255930 after retry RPC@1515132223) -> (continuation of Deferred@1283262923 after retry RPC@1857624982) -> (continuation of Deferred@1706472671 after retry RPC@1140155988))) after lookup
14:01:48.837 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:48.838 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] Successfully connected to peer
14:01:48.853 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xb5599936, L:/127.0.0.1:52122 - R:/127.13.247.67:36071] using TOKEN/null
14:01:48.875 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.66:46279
14:01:48.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:48.876130 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:46279
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36973
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.220924 16361 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.221266 16361 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.221351 16361 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:49.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.230001 16361 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:49.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.230180 16361 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.66
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.240870 16361 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.66:46279
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.66
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36973
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.66
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/logs
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.242769 16361 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:49.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.244746 16361 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:49.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.255156 16369 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:49.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.255352 16371 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:49.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:49.254973 16368 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:49.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.258323 16361 server_base.cc:1061] running on GCE node
14:01:49.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.258996 16361 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.260025 16361 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:49.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.261345 16361 hybrid_clock.cc:648] HybridClock initialized: now 1783519309261266 us; error 56 us; skew 500 ppm
14:01:49.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.261791 16361 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.264199 16361 webserver.cc:533] Webserver started at http://127.13.247.66:36973/ using document root <none> and password file <none>
14:01:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.264918 16361 fs_manager.cc:362] Metadata directory not provided
14:01:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.265009 16361 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.270467 16361 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.002s
14:01:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.273334 16377 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.274858 16361 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
14:01:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.275027 16361 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.275805 16361 fs_report.cc:389] FS layout report
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/wal
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/data
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.292487 16361 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:49.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.293452 16361 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:49.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.293707 16361 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.294761 16361 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.297191 16384 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.303508 16361 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.303609 16361 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:01:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.303715 16361 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:49.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.306885 16361 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:49.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.306982 16361 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
14:01:49.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.308283 16384 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap starting.
14:01:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.343644 16384 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Log is configured to *not* fsync() on all Append() calls
14:01:49.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.351778 16361 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.66:46279
14:01:49.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.351830 16491 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.66:46279 every 8 connection(s)
14:01:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.353550 16361 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-1/data/info.pb
14:01:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.353967 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 16361
14:01:49.354 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:49.398 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:49.398 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.67:36071
14:01:49.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:01:49.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.399394 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 15991
14:01:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.419384 16499 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.419777 16499 heartbeater.cc:461] Registering TS with master...
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.420318 16493 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.420496 16493 heartbeater.cc:461] Registering TS with master...
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.421082 16499 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.421291 16493 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.425084 16492 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.425216 16492 heartbeater.cc:461] Registering TS with master...
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.425727 16492 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.426035 14414 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.427050 14484 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.427333 14343 ts_manager.cc:194] Re-registered known tserver with Master: bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.428614 14484 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.66:43759
14:01:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.437386 16117 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.447007 16384 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.447595 16384 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Bootstrap complete.
14:01:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.448714 16384 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent bootstrapping tablet: real 0.142s	user 0.065s	sys 0.019s
14:01:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.452860 16384 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.453617 16384 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: bde3d574cb824a4aa08d2b3f34754f99, State: Initialized, Role: FOLLOWER
14:01:49.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.454288 16384 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.455533 16492 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:49.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.457711 16384 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99: Time spent starting tablet: real 0.007s	user 0.007s	sys 0.001s
14:01:49.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:49.542811 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 15991
14:01:49.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:49.567 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] handling channelInactive
14:01:49.567 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state READY due to: connection closed
14:01:49.573 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:49.574 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.574 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.575 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.596 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:49.596 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:49.596 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=3, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [8ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [8ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1739545631(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@263b30ec -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@56e781af -> wakeup thread Time-limited test)) after lookup
14:01:49.597 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:49.598 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.598 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.598 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.616 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:49.617 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:49.617 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=5, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [8ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [8ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [31ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [46ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1739545631(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@263b30ec -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@56e781af -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398))) after lookup
14:01:49.617 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:49.618 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.619 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.619 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:49.677 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:49.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=7, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=109), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [8ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [8ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [31ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [46ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [52ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [52ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [109ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1739545631(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@263b30ec -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@56e781af -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283))) after lookup
14:01:49.683 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:49.684 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.684 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.684 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.756 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:49.756 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:49.757 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=9, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=189), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [8ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [8ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [31ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [46ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [52ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [52ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [109ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [116ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [117ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [117ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [189ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1739545631(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@263b30ec -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@56e781af -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043))) after lookup
14:01:49.758 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:49.758 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.758 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.759 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:49.836 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:49.837 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=11, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=269), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [8ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [8ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [31ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [46ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [52ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [52ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [109ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [116ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [117ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [117ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [189ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [190ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [192ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [192ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [266ms] refreshing cache from master, [266ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [269ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1739545631(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@263b30ec -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043) -> (continuation of Deferred@1205714246 after retry RPC@1277362671), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@56e781af -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043) -> (continuation of Deferred@1205714246 after retry RPC@1277362671))) after lookup
14:01:49.837 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:49.838 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.838 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:49.838 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:50.736 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:50.736 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:50.736 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=13, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1169), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [8ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [8ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [31ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [46ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [52ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [52ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [109ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [116ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [117ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [117ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [189ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [190ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [192ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [192ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [266ms] refreshing cache from master, [266ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [269ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [270ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [271ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [271ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1166ms] refreshing cache from master, [1166ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1169ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1739545631(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@263b30ec -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043) -> (continuation of Deferred@1205714246 after retry RPC@1277362671) -> (continuation of Deferred@209837362 after retry RPC@1492748596), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@56e781af -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043) -> (continuation of Deferred@1205714246 after retry RPC@1277362671) -> (continuation of Deferred@209837362 after retry RPC@1492748596))) after lookup
14:01:50.737 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:50.738 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:50.738 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:50.738 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071
14:01:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.750656 16508 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:01:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.750923 16508 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:50.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.752657 16508 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:50.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:50.754372 16381 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:50.756018 16381 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:50.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.759588 16266 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "fafc4088d51349e7973eb5b714841aa8" is_pre_election: true
14:01:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.760509 16379 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99; no voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8
14:01:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.760954 16508 raft_consensus.cc:2749] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
14:01:50.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.786738 16513 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 27ce0884bf764cdbb025c6d3b9edc082)
14:01:50.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.786971 16513 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:50.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.787735 16513 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:50.790536 16200 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.795042 16446 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99" is_pre_election: true
14:01:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.795476 16446 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 8.
14:01:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.796113 16198 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082
14:01:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.796506 16513 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 9 FOLLOWER]: Leader pre-election won for term 10
14:01:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.796623 16513 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 27ce0884bf764cdbb025c6d3b9edc082)
14:01:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.796691 16513 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 9 FOLLOWER]: Advancing to term 10
14:01:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.798825 16513 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.799408 16513 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 10 election: Requested vote from peers 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.800033 16446 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "fafc4088d51349e7973eb5b714841aa8" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.800243 16446 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 8 FOLLOWER]: Advancing to term 10
14:01:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:50.801201 16200 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111)
14:01:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.804198 16446 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc4088d51349e7973eb5b714841aa8 in term 10.
14:01:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.804628 16198 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99, fafc4088d51349e7973eb5b714841aa8; no voters: 27ce0884bf764cdbb025c6d3b9edc082
14:01:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.804939 16513 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 10 FOLLOWER]: Leader election won for term 10
14:01:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.805505 16513 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 10 LEADER]: Becoming Leader. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Running, Role: LEADER
14:01:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.806059 16513 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:50.809811 14484 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 reported cstate change: term changed from 9 to 10, leader changed from 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67) to fafc4088d51349e7973eb5b714841aa8 (127.13.247.65). New cstate: current_term: 10 leader_uuid: "fafc4088d51349e7973eb5b714841aa8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:51.283833 16200 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 -> Peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Couldn't send request to peer 27ce0884bf764cdbb025c6d3b9edc082. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:51.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:51.345698 16446 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 10 FOLLOWER]: Refusing update from remote peer fafc4088d51349e7973eb5b714841aa8: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
14:01:51.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:51.346536 16513 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
14:01:53.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.079037 16200 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111) [suppressed 6 similar messages]
14:01:53.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:53.397 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:53.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=15, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=3830), Traces: [0ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [8ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [8ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [31ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [31ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [46ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [49ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [52ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [52ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [109ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [116ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [117ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [117ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [189ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [190ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [192ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [192ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [266ms] refreshing cache from master, [266ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [269ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [270ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [271ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [271ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1166ms] refreshing cache from master, [1166ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [1169ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [1170ms] sending RPC to server 27ce0884bf764cdbb025c6d3b9edc082, [1171ms] received response from server 27ce0884bf764cdbb025c6d3b9edc082: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [1171ms] delaying RPC due to: Network error: Failed to connect to peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071): Connection refused: /127.13.247.67:36071, [3827ms] refreshing cache from master, [3827ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [3830ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@1739545631(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@263b30ec -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043) -> (continuation of Deferred@1205714246 after retry RPC@1277362671) -> (continuation of Deferred@209837362 after retry RPC@1492748596) -> (continuation of Deferred@2121510286 after retry RPC@1784097629), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@56e781af -> wakeup thread Time-limited test -> (continuation of Deferred@1879301320 after retry RPC@444842398) -> (continuation of Deferred@1399967700 after retry RPC@1622684283) -> (continuation of Deferred@1689457796 after retry RPC@1161415043) -> (continuation of Deferred@1205714246 after retry RPC@1277362671) -> (continuation of Deferred@209837362 after retry RPC@1492748596) -> (continuation of Deferred@2121510286 after retry RPC@1784097629))) after lookup
14:01:53.399 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:53.400 [DEBUG - kudu-nio-0] (Connection.java:757) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] Successfully connected to peer
14:01:53.415 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xd0809204, L:/127.0.0.1:50186 - R:/127.13.247.65:44239] using TOKEN/null
14:01:53.441 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.67:36071
14:01:53.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.442868 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:36071
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:53.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43327
14:01:53.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:53.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:53.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:53.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:53.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.456472 16200 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 -> Peer 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071): Couldn't send request to peer 27ce0884bf764cdbb025c6d3b9edc082. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.67:36071: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:01:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.793083 16552 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.793486 16552 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.793607 16552 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:53.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.802366 16552 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:53.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.802616 16552 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.67
14:01:53.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.813412 16552 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:53.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:53.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:53.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:53.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.67:36071
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.67
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43327
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.67
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/logs
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.815573 16552 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.817557 16552 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.828779 16560 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.828804 16557 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:53.828753 16558 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.829658 16552 server_base.cc:1061] running on GCE node
14:01:53.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.830333 16552 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:53.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.831440 16552 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:53.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.832674 16552 hybrid_clock.cc:648] HybridClock initialized: now 1783519313832602 us; error 62 us; skew 500 ppm
14:01:53.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.833069 16552 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:53.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.835574 16552 webserver.cc:533] Webserver started at http://127.13.247.67:43327/ using document root <none> and password file <none>
14:01:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.836315 16552 fs_manager.cc:362] Metadata directory not provided
14:01:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.836407 16552 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.841997 16552 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.000s	sys 0.005s
14:01:53.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.844992 16566 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:53.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.846385 16552 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.846519 16552 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ce0884bf764cdbb025c6d3b9edc082"
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:18 on dist-test-slave-f6s0"
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.847203 16552 fs_report.cc:389] FS layout report
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/wal
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/data
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.868160 16552 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:53.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.869017 16552 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:53.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.869266 16552 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:53.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.870272 16552 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:53.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.872845 16573 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.879707 16552 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.879838 16552 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:01:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.879957 16552 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.883973 16552 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.884092 16552 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.004s	sys 0.000s
14:01:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.884637 16573 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap starting.
14:01:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.911269 16573 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Log is configured to *not* fsync() on all Append() calls
14:01:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.939626 16552 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.67:36071
14:01:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.939678 16680 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.67:36071 every 8 connection(s)
14:01:53.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.941603 16552 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-2/data/info.pb
14:01:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.952060 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 16552
14:01:53.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:53.989 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:53.989 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.13.247.65:44239
14:01:53.990 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.989573 16682 heartbeater.cc:344] Connected to a master server at 127.13.247.126:34459
14:01:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.989962 16682 heartbeater.cc:461] Registering TS with master...
14:01:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.990286 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 16182
14:01:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.991190 16682 heartbeater.cc:507] Master 127.13.247.126:34459 requested a full tablet report, sending...
14:01:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.992656 16573 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.993382 16573 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Bootstrap complete.
14:01:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:53.994908 16573 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent bootstrapping tablet: real 0.111s	user 0.052s	sys 0.020s
14:01:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.000999 16683 heartbeater.cc:344] Connected to a master server at 127.13.247.125:33751
14:01:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.001240 16683 heartbeater.cc:461] Registering TS with master...
14:01:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.002015 16683 heartbeater.cc:507] Master 127.13.247.125:33751 requested a full tablet report, sending...
14:01:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.004004 14413 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.004364 16573 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.005476 16573 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Initialized, Role: FOLLOWER
14:01:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.006338 16573 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.010041 14343 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:54.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.018455 16681 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:54.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.018617 16681 heartbeater.cc:461] Registering TS with master...
14:01:54.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.019304 16681 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.021376 14484 ts_manager.cc:194] Re-registered known tserver with Master: 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.023001 14484 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.67:40503
14:01:54.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.061153 16573 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082: Time spent starting tablet: real 0.066s	user 0.023s	sys 0.042s
14:01:54.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.063891 16681 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.064872 16307 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:54.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.067431 16636 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 9 FOLLOWER]: Advancing to term 10
14:01:54.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:54.189110 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 16182
14:01:54.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:54.208 [DEBUG - kudu-nio-0] (Connection.java:239) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] handling channelInactive
14:01:54.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state READY due to: connection closed
14:01:54.214 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:54.215 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.215 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.215 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:54.237 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:54.238 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=3, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [7ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [7ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@440158368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@6c89a2ef -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2240b21 -> wakeup thread Time-limited test)) after lookup
14:01:54.238 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:54.239 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.239 [DEBUG - kudu-nio-2] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.239 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.256 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:54.257 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:54.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=5, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [7ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [7ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [31ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [31ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [48ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@440158368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@6c89a2ef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2240b21 -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684))) after lookup
14:01:54.259 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:54.260 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.260 [DEBUG - kudu-nio-3] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.260 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.296 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:54.297 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:54.297 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=7, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [7ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [7ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [31ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [31ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [48ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [52ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [52ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@440158368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@6c89a2ef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2240b21 -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229))) after lookup
14:01:54.298 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:54.299 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.299 [DEBUG - kudu-nio-4] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.299 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:54.557 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:54.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=9, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=349), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [7ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [7ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [31ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [31ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [48ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [52ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [52ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [89ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [91ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [91ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [345ms] refreshing cache from master, [345ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [348ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@440158368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@6c89a2ef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229) -> (continuation of Deferred@1610668727 after retry RPC@931186293), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2240b21 -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229) -> (continuation of Deferred@1610668727 after retry RPC@931186293))) after lookup
14:01:54.558 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:54.559 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.560 [DEBUG - kudu-nio-5] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:54.560 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:55.197 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:55.197 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071),bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)[L]]}]
14:01:55.197 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=11, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=989), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [7ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [7ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [31ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [31ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [48ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [52ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [52ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [89ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [91ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [91ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [345ms] refreshing cache from master, [345ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [348ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [350ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [352ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [352ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [985ms] refreshing cache from master, [986ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [988ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@440158368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@6c89a2ef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229) -> (continuation of Deferred@1610668727 after retry RPC@931186293) -> (continuation of Deferred@1064549 after retry RPC@745807241), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2240b21 -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229) -> (continuation of Deferred@1610668727 after retry RPC@931186293) -> (continuation of Deferred@1064549 after retry RPC@745807241))) after lookup
14:01:55.198 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] connecting to peer
14:01:55.199 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:55.199 [DEBUG - kudu-nio-6] (Connection.java:709) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:55.199 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239) for tablet 48add68e49ac4613aed5d6771722e462: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239
14:01:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.373018 16703 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.373594 16703 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.374399 16703 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67:36071)
14:01:55.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:55.376726 16379 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:55.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.382159 16636 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "bde3d574cb824a4aa08d2b3f34754f99" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "27ce0884bf764cdbb025c6d3b9edc082" is_pre_election: true
14:01:55.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.383414 16381 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bde3d574cb824a4aa08d2b3f34754f99; no voters: 27ce0884bf764cdbb025c6d3b9edc082, fafc4088d51349e7973eb5b714841aa8
14:01:55.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.383778 16703 raft_consensus.cc:2749] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 10 FOLLOWER]: Leader pre-election lost for term 11. Reason: could not achieve majority
14:01:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.575291 16708 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.575525 16708 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.577044 16708 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:55.578372 16568 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:55.580497 16568 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.584824 16446 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99" is_pre_election: true
14:01:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.585053 16446 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 10.
14:01:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.585500 16568 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: fafc4088d51349e7973eb5b714841aa8
14:01:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.585829 16708 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 10 FOLLOWER]: Leader pre-election won for term 11
14:01:55.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.585983 16708 raft_consensus.cc:493] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 10 FOLLOWER]: Starting leader election (detected failure of leader fafc4088d51349e7973eb5b714841aa8)
14:01:55.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.586130 16708 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 10 FOLLOWER]: Advancing to term 11
14:01:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.588212 16708 raft_consensus.cc:515] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.588919 16708 leader_election.cc:290] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 11 election: Requested vote from peers fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239), bde3d574cb824a4aa08d2b3f34754f99 (127.13.247.66:46279)
14:01:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.589749 16446 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "48add68e49ac4613aed5d6771722e462" candidate_uuid: "27ce0884bf764cdbb025c6d3b9edc082" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "bde3d574cb824a4aa08d2b3f34754f99"
14:01:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.589988 16446 raft_consensus.cc:3060] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 10 FOLLOWER]: Advancing to term 11
14:01:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:55.590358 16568 leader_election.cc:336] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111)
14:01:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.592201 16446 raft_consensus.cc:2468] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 27ce0884bf764cdbb025c6d3b9edc082 in term 11.
14:01:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.592613 16568 leader_election.cc:304] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27ce0884bf764cdbb025c6d3b9edc082, bde3d574cb824a4aa08d2b3f34754f99; no voters: fafc4088d51349e7973eb5b714841aa8
14:01:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.592948 16708 raft_consensus.cc:2804] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 11 FOLLOWER]: Leader election won for term 11
14:01:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.593398 16708 raft_consensus.cc:697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [term 11 LEADER]: Becoming Leader. State: Replica: 27ce0884bf764cdbb025c6d3b9edc082, State: Running, Role: LEADER
14:01:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.593943 16708 consensus_queue.cc:237] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:55.597992 14484 catalog_manager.cc:5697] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 reported cstate change: term changed from 10 to 11, leader changed from fafc4088d51349e7973eb5b714841aa8 (127.13.247.65) to 27ce0884bf764cdbb025c6d3b9edc082 (127.13.247.67). New cstate: current_term: 11 leader_uuid: "27ce0884bf764cdbb025c6d3b9edc082" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } health_report { overall_health: UNKNOWN } } }
14:01:56.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.096716 16446 raft_consensus.cc:1275] T 48add68e49ac4613aed5d6771722e462 P bde3d574cb824a4aa08d2b3f34754f99 [term 11 FOLLOWER]: Refusing update from remote peer 27ce0884bf764cdbb025c6d3b9edc082: Log matching property violated. Preceding OpId in replica: term: 10 index: 40. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
14:01:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.099934 16726 consensus_queue.cc:1048] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 40, Time since last communication: 0.000s
14:01:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.100898 16568 consensus_peers.cc:597] T 48add68e49ac4613aed5d6771722e462 P 27ce0884bf764cdbb025c6d3b9edc082 -> Peer fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239): Couldn't send request to peer fafc4088d51349e7973eb5b714841aa8. Status: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:01:56.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 48add68e49ac4613aed5d6771722e462 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840' with partition [<start>, <end>)
14:01:56.397 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=48add68e49ac4613aed5d6771722e462@[27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)[L],bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279),fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)]}]
14:01:56.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=48add68e49ac4613aed5d6771722e462, attempt=13, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2189), Traces: [0ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [7ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [7ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [29ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [30ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [31ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [31ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [48ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [50ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [52ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [52ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [88ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [89ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [91ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [91ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [345ms] refreshing cache from master, [345ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [348ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [350ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [352ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [352ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [985ms] refreshing cache from master, [986ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [988ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, [990ms] sending RPC to server fafc4088d51349e7973eb5b714841aa8, [991ms] received response from server fafc4088d51349e7973eb5b714841aa8: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [991ms] delaying RPC due to: Network error: Failed to connect to peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239): Connection refused: /127.13.247.65:44239, [2186ms] refreshing cache from master, [2186ms] Sub RPC GetTableLocations: sending RPC to server master-127.13.247.124:36625, [2188ms] Sub RPC GetTableLocations: received response from server master-127.13.247.124:36625: OK, deferred=Deferred@440158368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$236/0x00007f1be8338f98@6c89a2ef -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229) -> (continuation of Deferred@1610668727 after retry RPC@931186293) -> (continuation of Deferred@1064549 after retry RPC@745807241) -> (continuation of Deferred@553380551 after retry RPC@753869572), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2240b21 -> wakeup thread Time-limited test -> (continuation of Deferred@648107370 after retry RPC@75382684) -> (continuation of Deferred@1498954109 after retry RPC@741310229) -> (continuation of Deferred@1610668727 after retry RPC@931186293) -> (continuation of Deferred@1064549 after retry RPC@745807241) -> (continuation of Deferred@553380551 after retry RPC@753869572))) after lookup
14:01:56.398 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] connecting to peer
14:01:56.399 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] Successfully connected to peer
14:01:56.425 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x5392f97e, L:/127.0.0.1:55948 - R:/127.13.247.67:36071] using TOKEN/null
14:01:56.447 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.13.247.65:44239
14:01:56.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.448498 14301 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu
14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:01:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:01:56.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.802364 16737 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:01:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.802743 16737 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:01:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.802876 16737 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:01:56.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.811669 16737 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:01:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.811924 16737 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.247.65
14:01:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.822520 16737 tablet_server_runner.cc:78] Tablet server non-default flags:
14:01:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.247.84:32967
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.247.65:44239
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.247.65
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32823
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.247.126:34459,127.13.247.125:33751,127.13.247.124:36625
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.247.65
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/logs
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 844bedc15e81bd62879377e001142786f7d931d8
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 13:43:23 UTC on bdcb31816ec0
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12774
14:01:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:01:56.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.824672 16737 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:01:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.826610 16737 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:01:56.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.836316 16744 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:56.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.836342 16745 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:56.837571 16747 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:01:56.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.838287 16737 server_base.cc:1061] running on GCE node
14:01:56.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.838976 16737 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:01:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.840019 16737 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:01:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.841248 16737 hybrid_clock.cc:648] HybridClock initialized: now 1783519316841204 us; error 35 us; skew 500 ppm
14:01:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.841601 16737 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:01:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.843992 16737 webserver.cc:533] Webserver started at http://127.13.247.65:32823/ using document root <none> and password file <none>
14:01:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.844787 16737 fs_manager.cc:362] Metadata directory not provided
14:01:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.844923 16737 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:01:56.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.850586 16737 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.003s
14:01:56.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.853637 16753 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:01:56.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.855087 16737 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.002s
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.855297 16737 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data,/tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc4088d51349e7973eb5b714841aa8"
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:01:17 on dist-test-slave-f6s0"
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.856076 16737 fs_report.cc:389] FS layout report
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/wal
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/data
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:01:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:01:56.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.880584 16737 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:01:56.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.881624 16737 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:01:56.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.882009 16737 kserver.cc:163] Server-wide thread pool size limit: 3276
14:01:56.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.883176 16737 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:01:56.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.885617 16760 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:01:56.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.892489 16737 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:01:56.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.892614 16737 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
14:01:56.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.892753 16737 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:01:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.896044 16737 ts_tablet_manager.cc:616] Registered 1 tablets
14:01:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.896199 16737 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
14:01:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.896730 16760 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap starting.
14:01:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.930177 16760 log.cc:826] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Log is configured to *not* fsync() on all Append() calls
14:01:56.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.945086 16737 rpc_server.cc:307] RPC server started. Bound to: 127.13.247.65:44239
14:01:56.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.945523 16867 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.247.65:44239 every 8 connection(s)
14:01:56.948 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:01:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.946842 16737 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasktTzzjk/test-tmp/mini-kudu-cluster2592797402462039027/ts-0/data/info.pb
14:01:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.947852 14301 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu as pid 16737
14:01:56.989 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:56.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.992568 16760 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:01:56.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.993494 16760 tablet_bootstrap.cc:492] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Bootstrap complete.
14:01:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:56.995714 16760 ts_tablet_manager.cc:1403] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent bootstrapping tablet: real 0.099s	user 0.079s	sys 0.017s
14:01:56.997 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783519278840, tablet=48add68e49ac4613aed5d6771722e462, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:01:57.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.007180 16760 raft_consensus.cc:359] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:57.012 [DEBUG - main] (Connection.java:575) [peer master-127.13.247.124:36625(127.13.247.124:36625)] disconnecting while in state READY
14:01:57.014 [DEBUG - main] (Connection.java:575) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] disconnecting while in state READY
14:01:57.014 [DEBUG - main] (Connection.java:575) [peer master-127.13.247.126:34459(127.13.247.126:34459)] disconnecting while in state READY
14:01:57.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.015738 16760 raft_consensus.cc:740] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc4088d51349e7973eb5b714841aa8, State: Initialized, Role: FOLLOWER
14:01:57.016 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.247.124:36625(127.13.247.124:36625)] handling channelInactive
14:01:57.016 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.247.124:36625(127.13.247.124:36625)] cleaning up while in state READY due to: connection closed
14:01:57.016 [DEBUG - main] (Connection.java:575) [peer bde3d574cb824a4aa08d2b3f34754f99(127.13.247.66:46279)] disconnecting while in state TERMINATED
14:01:57.017 [DEBUG - main] (Connection.java:575) [peer master-127.13.247.125:33751(127.13.247.125:33751)] disconnecting while in state READY
14:01:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.012506 16868 heartbeater.cc:344] Connected to a master server at 127.13.247.124:36625
14:01:57.017 [DEBUG - main] (Connection.java:575) [peer fafc4088d51349e7973eb5b714841aa8(127.13.247.65:44239)] disconnecting while in state TERMINATED
14:01:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.017438 16868 heartbeater.cc:461] Registering TS with master...
14:01:57.017 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.13.247.125:33751(127.13.247.125:33751)] handling channelInactive
14:01:57.017 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.13.247.125:33751(127.13.247.125:33751)] cleaning up while in state READY due to: connection closed
14:01:57.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.018335 16760 consensus_queue.cc:260] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc4088d51349e7973eb5b714841aa8" member_type: VOTER last_known_addr { host: "127.13.247.65" port: 44239 } } peers { permanent_uuid: "27ce0884bf764cdbb025c6d3b9edc082" member_type: VOTER last_known_addr { host: "127.13.247.67" port: 36071 } } peers { permanent_uuid: "bde3d574cb824a4aa08d2b3f34754f99" member_type: VOTER last_known_addr { host: "127.13.247.66" port: 46279 } }
14:01:57.019 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.247.126:34459(127.13.247.126:34459)] handling channelInactive
14:01:57.019 [DEBUG - kudu-nio-7] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:01:57.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.247.126:34459(127.13.247.126:34459)] cleaning up while in state READY due to: connection closed
14:01:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.018777 16868 heartbeater.cc:507] Master 127.13.247.124:36625 requested a full tablet report, sending...
14:01:57.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.023438 14484 ts_manager.cc:194] Re-registered known tserver with Master: fafc4088d51349e7973eb5b714841aa8 (127.13.247.65:44239)
14:01:57.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.025328 14484 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.247.65:33989
14:01:57.026 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] handling channelInactive
14:01:57.026 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 27ce0884bf764cdbb025c6d3b9edc082(127.13.247.67:36071)] cleaning up while in state READY due to: connection closed
14:01:57.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.037701 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 16737
14:01:57.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.061228 16863 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:57.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.062644 16760 ts_tablet_manager.cc:1434] T 48add68e49ac4613aed5d6771722e462 P fafc4088d51349e7973eb5b714841aa8: Time spent starting tablet: real 0.067s	user 0.014s	sys 0.020s
14:01:57.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.065971 16868 heartbeater.cc:499] Master 127.13.247.124:36625 was elected leader, sending a full tablet report...
14:01:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.164089 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 16737
14:01:57.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:57.187708 14398 connection.cc:573] server connection from 127.13.247.65:32937 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:57.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:57.188346 14469 connection.cc:573] server connection from 127.13.247.65:33989 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:57.188832 16738 negotiation.cc:336] Failed RPC negotiation. Trace:
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.104410 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.13.247.65:44239 (local address 127.13.247.67:34881)
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.104618 (+   208us) negotiation.cc:107] Waiting for socket to connect
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.104640 (+    22us) client_negotiation.cc:175] Beginning negotiation
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.104792 (+   152us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.187846 (+ 83054us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.13.247.65:44239: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":66}
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:01:57.188895 16877 negotiation.cc:336] Failed RPC negotiation. Trace:
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.005613 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.13.247.65:45101 (local address 127.13.247.126:34459)
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.053816 (+ 48203us) server_negotiation.cc:207] Beginning negotiation
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.053824 (+     8us) server_negotiation.cc:400] Waiting for connection header
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.053845 (+    21us) server_negotiation.cc:408] Connection header received
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.053910 (+    65us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.053916 (+     6us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.053998 (+    82us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.054114 (+   116us) server_negotiation.cc:227] Negotiated authn=SASL
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.059313 (+  5199us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.060723 (+  1410us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:01:57.187948 (+127225us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.247.65:45101: BlockingRecv error: recv got EOF from 127.13.247.65:45101 (error 108)
14:01:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":48082,"thread_start_us":47998,"threads_started":1}
14:01:57.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.190086 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 16361
14:01:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.197237 16487 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.293212 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 16361
14:01:57.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.313797 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 16552
14:01:57.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.322078 16676 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:57.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.445106 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 16552
14:01:57.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.465641 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 14312
14:01:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.466910 14373 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:57.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.583750 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 14312
14:01:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.605391 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 14380
14:01:57.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.606527 14444 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:57.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.723202 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 14380
14:01:57.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.746227 14301 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu pid 14451
14:01:57.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.747495 14515 generic_service.cc:196] Checking for leaks (request via RPC)
14:01:57.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:01:57.868240 14301 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasktTzzjk/build/asan/bin/kudu with pid 14451
14:01:57.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T14:01:57Z chronyd exiting

Time: 44.282

OK (1 test)

