JUnit version 4.13.2
.00:20:05.406 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1786494005397
00:20:05.474 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
00:20:05.512 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:20:05.529 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_i0RbK/test-tmp
00:20:05.557 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
00:20:05.560 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_i0RbK/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
00:20:05.778 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

00:20:05.984 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:20:05.985 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

00:20:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:05.989395 22641 mini_oidc.cc:150] Starting JWKS server
00:20:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:05.992879 22641 webserver.cc:533] Webserver started at http://127.0.0.1:33391/ using document root <none> and password file <none>
00:20:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:05.993233 22641 mini_oidc.cc:186] Starting OIDC Discovery server
00:20:05.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:05.996197 22641 webserver.cc:533] Webserver started at http://127.0.0.1:38423/ using document root <none> and password file <none>
00:20:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:20:06Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
00:20:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:20:06Z Disabled control of system clock
00:20:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.037990 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/wal
00:20:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data
00:20:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/logs
00:20:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data/info.pb
00:20:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.126:40901
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.126
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:20:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:20:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33391/jwks.json with env {}
00:20:06.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.410458 22655 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:06.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.410830 22655 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:06.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.410912 22655 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.420099 22655 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:20:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.420306 22655 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.420432 22655 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:20:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.420499 22655 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:20:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.420576 22655 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:20:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.420625 22655 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33391/jwks.json
00:20:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.431794 22655 master_runner.cc:387] Master server non-default flags:
00:20:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/logs
00:20:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/wal
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.126:40901
00:20:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:20:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:20:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33391/jwks.json
00:20:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data/info.pb
00:20:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.126
00:20:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:20:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.434087 22655 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.436599 22655 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:06.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.446517 22660 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:06.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.446545 22661 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:06.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.447132 22663 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.448902 22655 server_base.cc:1061] running on GCE node
00:20:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.449862 22655 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.451432 22655 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.452656 22655 hybrid_clock.cc:648] HybridClock initialized: now 1786494006452587 us; error 66 us; skew 500 ppm
00:20:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.452984 22655 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.455248 22655 webserver.cc:533] Webserver started at http://127.22.28.126:41549/ using document root <none> and password file <none>
00:20:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.456017 22655 fs_manager.cc:362] Metadata directory not provided
00:20:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.456110 22655 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.456607 22655 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:20:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.459151 22655 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data/instance:
00:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c1645ca3be24cc191ca9d51e073624c"
00:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:06 on dist-test-slave-8w9v"
00:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.460057 22655 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/wal/instance:
00:20:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c1645ca3be24cc191ca9d51e073624c"
00:20:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:06 on dist-test-slave-8w9v"
00:20:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.467096 22655 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.004s	sys 0.002s
00:20:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.471076 22669 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.472858 22655 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
00:20:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.473089 22655 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/wal
00:20:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c1645ca3be24cc191ca9d51e073624c"
00:20:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:06 on dist-test-slave-8w9v"
00:20:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.473380 22655 fs_report.cc:389] FS layout report
00:20:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/wal
00:20:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/wal
00:20:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data/data
00:20:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:06.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:06.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:06.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.503998 22655 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:06.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.505286 22655 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:06.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.505657 22655 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:06.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.527946 22655 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.126:40901
00:20:06.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.527966 22737 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.126:40901 every 8 connection(s)
00:20:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.529968 22655 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/data/info.pb
00:20:06.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.533610 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 22655
00:20:06.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.533977 22641 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-0/wal/instance
00:20:06.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.534812 22738 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:20:06.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.536777 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:06.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:06.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/wal
00:20:06.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data
00:20:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/logs
00:20:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data/info.pb
00:20:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:06.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:06.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:06.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:06.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:06.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:06.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:06.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:20:06.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:06.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:20:06.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:20:06.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.125:43953
00:20:06.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.125
00:20:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:20:06.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:06.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:20:06.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33391/jwks.json with env {}
00:20:06.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.542126 22738 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } has no permanent_uuid. Determining permanent_uuid...
00:20:06.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.561729 22738 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } has no permanent_uuid. Determining permanent_uuid...
00:20:06.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.563396 22671 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.125:43953: connect: Connection refused (error 111)
00:20:06.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.565449 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.125:43953: Network error: Client connection negotiation failed: client connection to 127.22.28.125:43953: connect: Connection refused (error 111)
00:20:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.614878 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } attempt: 1
00:20:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.617483 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.125:43953: Network error: Client connection negotiation failed: client connection to 127.22.28.125:43953: connect: Connection refused (error 111)
00:20:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.685813 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } attempt: 2
00:20:06.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.688248 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.125:43953: Network error: Client connection negotiation failed: client connection to 127.22.28.125:43953: connect: Connection refused (error 111)
00:20:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.779600 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } attempt: 3
00:20:06.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.782171 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.125:43953: Network error: Client connection negotiation failed: client connection to 127.22.28.125:43953: connect: Connection refused (error 111)
00:20:06.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.910212 22742 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.910632 22742 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.910750 22742 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:06.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.919714 22742 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:20:06.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.919883 22742 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:06.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.919967 22742 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:20:06.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.920032 22742 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:20:06.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.920113 22742 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:20:06.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.920202 22742 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33391/jwks.json
00:20:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.925469 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } attempt: 4
00:20:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.928041 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.125:43953: Network error: Client connection negotiation failed: client connection to 127.22.28.125:43953: connect: Connection refused (error 111)
00:20:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.933015 22742 master_runner.cc:387] Master server non-default flags:
00:20:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/logs
00:20:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/wal
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.125:43953
00:20:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:20:06.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:06.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:20:06.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33391/jwks.json
00:20:06.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:06.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:06.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data/info.pb
00:20:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.125
00:20:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:20:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:06.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:06.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:06.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.935111 22742 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:06.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.937315 22742 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:06.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.946950 22754 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.949303 22756 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:06.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:06.950080 22759 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.951095 22742 server_base.cc:1061] running on GCE node
00:20:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.952181 22742 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:06.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.953984 22742 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.955248 22742 hybrid_clock.cc:648] HybridClock initialized: now 1786494006955177 us; error 60 us; skew 500 ppm
00:20:06.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.955968 22742 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:06.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.958650 22742 webserver.cc:533] Webserver started at http://127.22.28.125:44931/ using document root <none> and password file <none>
00:20:06.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.959544 22742 fs_manager.cc:362] Metadata directory not provided
00:20:06.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.959652 22742 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:06.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.960079 22742 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:20:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.963804 22742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data/instance:
00:20:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6aa5364b59e945b381f70b707700ca1a"
00:20:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:06 on dist-test-slave-8w9v"
00:20:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.965029 22742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/wal/instance:
00:20:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6aa5364b59e945b381f70b707700ca1a"
00:20:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:06 on dist-test-slave-8w9v"
00:20:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.973304 22742 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.002s	sys 0.005s
00:20:06.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.978909 22767 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:06.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.981068 22742 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
00:20:06.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.981340 22742 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/wal
00:20:06.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6aa5364b59e945b381f70b707700ca1a"
00:20:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:06 on dist-test-slave-8w9v"
00:20:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:06.981658 22742 fs_report.cc:389] FS layout report
00:20:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/wal
00:20:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/wal
00:20:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data/data
00:20:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.005190 22742 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.006250 22742 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.006587 22742 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:07.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.029479 22742 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.125:43953
00:20:07.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.029500 22822 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.125:43953 every 8 connection(s)
00:20:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.031328 22742 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/data/info.pb
00:20:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.032661 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 22742
00:20:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.032848 22641 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-1/wal/instance
00:20:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.034152 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/wal
00:20:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data
00:20:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/logs
00:20:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data/info.pb
00:20:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:20:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:20:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:20:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.124:44295
00:20:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.124
00:20:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:20:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:20:07.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33391/jwks.json with env {}
00:20:07.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.037587 22823 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:20:07.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.046432 22823 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } has no permanent_uuid. Determining permanent_uuid...
00:20:07.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.062676 22823 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } has no permanent_uuid. Determining permanent_uuid...
00:20:07.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.073642 22823 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } has no permanent_uuid. Determining permanent_uuid...
00:20:07.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.074913 22768 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.076894 22823 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.126271 22823 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 1
00:20:07.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.128830 22823 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.197121 22823 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 2
00:20:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.199667 22823 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.227391 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } attempt: 5
00:20:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.239059 22738 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } has no permanent_uuid. Determining permanent_uuid...
00:20:07.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.241422 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.290987 22823 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 3
00:20:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.292737 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 1
00:20:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.293676 22823 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.294909 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.363173 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 2
00:20:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.365609 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.417948 22826 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.418406 22826 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.418529 22826 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.427546 22826 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:20:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.427704 22826 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.427802 22826 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:20:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.427863 22826 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:20:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.427942 22826 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:20:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.427992 22826 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33391/jwks.json
00:20:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.436941 22823 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 4
00:20:07.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.439293 22823 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.440368 22826 master_runner.cc:387] Master server non-default flags:
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/logs
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data
00:20:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/wal
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.124:44295
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33391/jwks.json
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data/info.pb
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.124
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.442318 22826 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.445003 22826 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:07.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.454787 22839 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:07.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.455080 22836 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:07.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.454818 22837 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:07.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.456171 22826 server_base.cc:1061] running on GCE node
00:20:07.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.457052 22826 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.458561 22826 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:07.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.459795 22826 hybrid_clock.cc:648] HybridClock initialized: now 1786494007459719 us; error 73 us; skew 500 ppm
00:20:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.460175 22826 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.462608 22826 webserver.cc:533] Webserver started at http://127.22.28.124:37597/ using document root <none> and password file <none>
00:20:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.463529 22826 fs_manager.cc:362] Metadata directory not provided
00:20:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.463671 22826 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.464196 22826 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:20:07.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.466920 22826 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data/instance:
00:20:07.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef34a573c868405786f657e50700508f"
00:20:07.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:07 on dist-test-slave-8w9v"
00:20:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.467921 22826 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/wal/instance:
00:20:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef34a573c868405786f657e50700508f"
00:20:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:07 on dist-test-slave-8w9v"
00:20:07.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.471866 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 3
00:20:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.474975 22738 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.28.124:44295: Network error: Client connection negotiation failed: client connection to 127.22.28.124:44295: connect: Connection refused (error 111)
00:20:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.477840 22826 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.007s	sys 0.001s
00:20:07.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.482200 22846 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.483875 22826 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.000s
00:20:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.484107 22826 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/wal
00:20:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef34a573c868405786f657e50700508f"
00:20:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:07 on dist-test-slave-8w9v"
00:20:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.484444 22826 fs_report.cc:389] FS layout report
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/wal
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/wal
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data/data
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.516363 22826 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:07.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.517750 22826 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:07.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.518179 22826 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.540930 22826 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.124:44295
00:20:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.540947 22909 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.124:44295 every 8 connection(s)
00:20:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.542937 22826 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/data/info.pb
00:20:07.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.547871 22910 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:20:07.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.548032 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 22826
00:20:07.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.548269 22641 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/master-2/wal/instance
00:20:07.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.554390 22910 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } has no permanent_uuid. Determining permanent_uuid...
00:20:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.569903 22910 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } has no permanent_uuid. Determining permanent_uuid...
00:20:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.578541 22910 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } has no permanent_uuid. Determining permanent_uuid...
00:20:07.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.591252 22910 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: Bootstrap starting.
00:20:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.595269 22910 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: Neither blocks nor log segments found. Creating new log.
00:20:07.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.596733 22910 log.cc:826] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: Log is configured to *not* fsync() on all Append() calls
00:20:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.599866 22910 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: No bootstrap required, opened a new log
00:20:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.605854 22910 raft_consensus.cc:359] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.606324 22910 raft_consensus.cc:385] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:20:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.606496 22910 raft_consensus.cc:740] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef34a573c868405786f657e50700508f, State: Initialized, Role: FOLLOWER
00:20:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.607251 22910 consensus_queue.cc:260] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.608295 22919 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.608589 22919 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [sys.catalog]: This master's current role is: FOLLOWER
00:20:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.609426 22910 sys_catalog.cc:565] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [sys.catalog]: configured and running, proceeding with master startup.
00:20:07.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.620863 22931 catalog_manager.cc:1594] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:20:07.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.621038 22931 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:20:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.617079 22826 master_runner.cc:187] Error getting master registration for 127.22.28.126:40901: OK, instance_id { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" instance_seqno: 1786494006522087 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:20:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.623224 22826 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:20:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.652235 22738 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 4
00:20:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.664296 22738 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c: Bootstrap starting.
00:20:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.667810 22738 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c: Neither blocks nor log segments found. Creating new log.
00:20:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.669111 22738 log.cc:826] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c: Log is configured to *not* fsync() on all Append() calls
00:20:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.672554 22738 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c: No bootstrap required, opened a new log
00:20:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.679272 22738 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.679598 22738 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:20:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.679715 22738 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c1645ca3be24cc191ca9d51e073624c, State: Initialized, Role: FOLLOWER
00:20:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.680506 22738 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.681713 22935 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.682397 22935 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: This master's current role is: FOLLOWER
00:20:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.683327 22738 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: configured and running, proceeding with master startup.
00:20:07.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.699585 22946 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:20:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.701050 22946 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:20:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.695780 22655 master_runner.cc:187] Error getting master registration for 127.22.28.125:43953: OK, instance_id { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" instance_seqno: 1786494007023454 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:20:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.707615 22655 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:20:07.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.738592 22823 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } attempt: 5
00:20:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.744412 22935 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:20:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.744967 22935 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.750901 22935 leader_election.cc:290] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6aa5364b59e945b381f70b707700ca1a (127.22.28.125:43953), ef34a573c868405786f657e50700508f (127.22.28.124:44295)
00:20:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.756018 22876 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1645ca3be24cc191ca9d51e073624c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef34a573c868405786f657e50700508f" is_pre_election: true
00:20:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.756795 22876 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0c1645ca3be24cc191ca9d51e073624c in term 0.
00:20:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.756932 22823 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a: Bootstrap starting.
00:20:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.758507 22670 leader_election.cc:304] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0c1645ca3be24cc191ca9d51e073624c, ef34a573c868405786f657e50700508f; no voters: 
00:20:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.759039 22678 tablet.cc:2406] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:20:07.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.759096 22935 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Leader pre-election won for term 1
00:20:07.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.759275 22935 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:20:07.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.759382 22935 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 0 FOLLOWER]: Advancing to term 1
00:20:07.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.761656 22823 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a: Neither blocks nor log segments found. Creating new log.
00:20:07.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.762773 22935 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.763100 22823 log.cc:826] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a: Log is configured to *not* fsync() on all Append() calls
00:20:07.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.766096 22876 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1645ca3be24cc191ca9d51e073624c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef34a573c868405786f657e50700508f"
00:20:07.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.766417 22876 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [term 0 FOLLOWER]: Advancing to term 1
00:20:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.769814 22935 leader_election.cc:290] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [CANDIDATE]: Term 1 election: Requested vote from peers 6aa5364b59e945b381f70b707700ca1a (127.22.28.125:43953), ef34a573c868405786f657e50700508f (127.22.28.124:44295)
00:20:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:07.770789 22852 tablet.cc:2406] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:20:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.772163 22876 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c1645ca3be24cc191ca9d51e073624c in term 1.
00:20:07.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.773119 22670 leader_election.cc:304] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0c1645ca3be24cc191ca9d51e073624c, ef34a573c868405786f657e50700508f; no voters: 
00:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.774015 22935 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 1 FOLLOWER]: Leader election won for term 1
00:20:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.777489 22823 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a: No bootstrap required, opened a new log
00:20:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.777676 22935 raft_consensus.cc:697] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [term 1 LEADER]: Becoming Leader. State: Replica: 0c1645ca3be24cc191ca9d51e073624c, State: Running, Role: LEADER
00:20:07.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.778367 22935 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.788015 22823 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.789798 22823 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:20:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.790588 22823 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6aa5364b59e945b381f70b707700ca1a, State: Initialized, Role: FOLLOWER
00:20:07.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.792234 22823 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } }
00:20:07.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.801512 22949 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0c1645ca3be24cc191ca9d51e073624c. Latest consensus state: current_term: 1 leader_uuid: "0c1645ca3be24cc191ca9d51e073624c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.802034 22949 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: This master's current role is: LEADER
00:20:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.810343 22952 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
00:20:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.810570 22951 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.811160 22951 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [sys.catalog]: This master's current role is: FOLLOWER
00:20:07.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.814148 22823 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [sys.catalog]: configured and running, proceeding with master startup.
00:20:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.814571 22795 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1645ca3be24cc191ca9d51e073624c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6aa5364b59e945b381f70b707700ca1a"
00:20:07.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.815008 22795 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [term 0 FOLLOWER]: Advancing to term 1
00:20:07.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.815430 22952 catalog_manager.cc:1520] Initializing Kudu cluster ID...
00:20:07.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.818677 22795 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c1645ca3be24cc191ca9d51e073624c in term 1.
00:20:07.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.814229 22796 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1645ca3be24cc191ca9d51e073624c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6aa5364b59e945b381f70b707700ca1a" is_pre_election: true
00:20:07.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.819710 22796 raft_consensus.cc:2376] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 0c1645ca3be24cc191ca9d51e073624c in term 1. Re-sending same reply.
00:20:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.834888 22876 raft_consensus.cc:1275] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [term 1 FOLLOWER]: Refusing update from remote peer 0c1645ca3be24cc191ca9d51e073624c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:20:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.837525 22949 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:20:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.847194 22796 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [term 1 FOLLOWER]: Refusing update from remote peer 0c1645ca3be24cc191ca9d51e073624c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:20:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.848690 22949 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.863788 22919 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0c1645ca3be24cc191ca9d51e073624c. Latest consensus state: current_term: 1 leader_uuid: "0c1645ca3be24cc191ca9d51e073624c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.864189 22919 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [sys.catalog]: This master's current role is: FOLLOWER
00:20:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.869408 22954 mvcc.cc:204] Tried to move back new op lower bound from 7317479456073531392 to 7317479455871213568. Current Snapshot: MvccSnapshot[applied={T|T < 7317479456073531392}]
00:20:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.873608 22919 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0c1645ca3be24cc191ca9d51e073624c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.873906 22919 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f [sys.catalog]: This master's current role is: FOLLOWER
00:20:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.877128 22951 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0c1645ca3be24cc191ca9d51e073624c. Latest consensus state: current_term: 1 leader_uuid: "0c1645ca3be24cc191ca9d51e073624c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.877395 22951 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [sys.catalog]: This master's current role is: FOLLOWER
00:20:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.880896 22949 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0c1645ca3be24cc191ca9d51e073624c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.881278 22949 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: This master's current role is: LEADER
00:20:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.882346 22949 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0c1645ca3be24cc191ca9d51e073624c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.882613 22949 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c [sys.catalog]: This master's current role is: LEADER
00:20:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.883553 22952 catalog_manager.cc:1383] Generated new cluster ID: 45f1f011e337406e962a09314b2a6270
00:20:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.883673 22952 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
00:20:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.896692 22951 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0c1645ca3be24cc191ca9d51e073624c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c1645ca3be24cc191ca9d51e073624c" member_type: VOTER last_known_addr { host: "127.22.28.126" port: 40901 } } peers { permanent_uuid: "6aa5364b59e945b381f70b707700ca1a" member_type: VOTER last_known_addr { host: "127.22.28.125" port: 43953 } } peers { permanent_uuid: "ef34a573c868405786f657e50700508f" member_type: VOTER last_known_addr { host: "127.22.28.124" port: 44295 } } }
00:20:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.896991 22951 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a [sys.catalog]: This master's current role is: FOLLOWER
00:20:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.913861 22952 catalog_manager.cc:1406] Generated new certificate authority record
00:20:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.915867 22952 catalog_manager.cc:1540] Loading token signing keys...
00:20:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.948559 22952 catalog_manager.cc:6092] T 00000000000000000000000000000000 P 0c1645ca3be24cc191ca9d51e073624c: Generated new TSK 0
00:20:07.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.950373 22952 catalog_manager.cc:1550] Initializing in-progress tserver states...
00:20:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.972012 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:0
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:07.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.988919 22973 catalog_manager.cc:1295] Loaded cluster ID: 45f1f011e337406e962a09314b2a6270
00:20:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.989574 22973 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a: loading cluster ID for follower catalog manager: success
00:20:07.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.994702 22973 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a: acquiring CA information for follower catalog manager: success
00:20:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:07.998409 22973 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 6aa5364b59e945b381f70b707700ca1a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:20:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.489292 22975 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:08.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.489707 22975 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:08.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.489822 22975 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:08.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.498911 22975 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:08.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.499166 22975 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.65
00:20:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.510215 22975 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:08.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:08.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:0
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.512727 22975 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.515576 22975 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:08.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.528223 22981 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.532588 22982 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:08.553318 22985 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.553720 22975 server_base.cc:1061] running on GCE node
00:20:08.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.555259 22975 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:08.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.556666 22975 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:08.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.558204 22975 hybrid_clock.cc:648] HybridClock initialized: now 1786494008558084 us; error 110 us; skew 500 ppm
00:20:08.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.558612 22975 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:08.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.561609 22975 webserver.cc:533] Webserver started at http://127.22.28.65:44085/ using document root <none> and password file <none>
00:20:08.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.562630 22975 fs_manager.cc:362] Metadata directory not provided
00:20:08.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.562784 22975 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:08.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.563186 22975 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:20:08.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.566633 22975 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/instance:
00:20:08.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:08.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:08 on dist-test-slave-8w9v"
00:20:08.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.567749 22975 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal/instance:
00:20:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:08 on dist-test-slave-8w9v"
00:20:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.574679 22975 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.001s	sys 0.004s
00:20:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.578730 22991 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.580467 22975 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:20:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.580647 22975 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:08 on dist-test-slave-8w9v"
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.580845 22975 fs_report.cc:389] FS layout report
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/data
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:08.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.605324 22975 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:08.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.606220 22975 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:08.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.606490 22975 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:08.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.607486 22975 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.609486 22975 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:20:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.609580 22975 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:20:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.609658 22975 ts_tablet_manager.cc:616] Registered 0 tablets
00:20:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.609710 22975 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:20:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.624006 22931 catalog_manager.cc:1295] Loaded cluster ID: 45f1f011e337406e962a09314b2a6270
00:20:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.624176 22931 catalog_manager.cc:1588] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: loading cluster ID for follower catalog manager: success
00:20:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.628500 22931 catalog_manager.cc:1610] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: acquiring CA information for follower catalog manager: success
00:20:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.630739 22931 catalog_manager.cc:1638] T 00000000000000000000000000000000 P ef34a573c868405786f657e50700508f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:20:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.649827 22975 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.65:33753
00:20:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.649863 23124 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.65:33753 every 8 connection(s)
00:20:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.651278 22975 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.653157 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 22975
00:20:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.653367 22641 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal/instance
00:20:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.661643 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:0
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:08.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.684286 23126 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.684736 23127 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.684860 23126 heartbeater.cc:461] Registering TS with master...
00:20:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.684902 23127 heartbeater.cc:461] Registering TS with master...
00:20:08.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.686064 23126 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:08.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.686571 23127 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.689627 22785 ts_manager.cc:194] Registered new tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.689728 22691 ts_manager.cc:194] Registered new tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:08.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.692005 23125 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:08.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.692224 23125 heartbeater.cc:461] Registering TS with master...
00:20:08.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.692332 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.65:40863
00:20:08.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.692831 23125 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:08.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:08.695053 22864 ts_manager.cc:194] Registered new tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:09.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.090041 23136 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:09.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.090389 23136 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:09.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.090476 23136 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:09.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.100860 23136 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:09.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.101038 23136 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.66
00:20:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.111411 23136 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:0
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:09.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.113252 23136 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:09.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.115298 23136 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.126385 23143 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.126616 23144 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.127537 23146 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:09.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.128911 23136 server_base.cc:1061] running on GCE node
00:20:09.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.129529 23136 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.130548 23136 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:09.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.131812 23136 hybrid_clock.cc:648] HybridClock initialized: now 1786494009131688 us; error 107 us; skew 500 ppm
00:20:09.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.132285 23136 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.134845 23136 webserver.cc:533] Webserver started at http://127.22.28.66:41155/ using document root <none> and password file <none>
00:20:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.135753 23136 fs_manager.cc:362] Metadata directory not provided
00:20:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.135902 23136 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.136423 23136 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:20:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.139223 23136 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/instance:
00:20:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.140259 23136 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal/instance:
00:20:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.146469 23136 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.000s
00:20:09.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.150664 23155 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:09.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.152338 23136 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
00:20:09.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.152551 23136 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.152895 23136 fs_report.cc:389] FS layout report
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/data
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.181051 23136 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.182163 23136 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.182528 23136 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:09.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.184525 23136 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:09.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.186559 23136 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:20:09.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.186684 23136 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:20:09.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.186815 23136 ts_tablet_manager.cc:616] Registered 0 tablets
00:20:09.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.186896 23136 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:20:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.223546 23136 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.66:40559
00:20:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.223593 23284 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.66:40559 every 8 connection(s)
00:20:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.225454 23136 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:09.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.232846 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 23136
00:20:09.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.237412 22641 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal/instance
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.241724 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:0
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:09.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.261658 23285 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:09.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.261797 23287 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:09.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.262176 23287 heartbeater.cc:461] Registering TS with master...
00:20:09.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.263458 23287 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.262173 23285 heartbeater.cc:461] Registering TS with master...
00:20:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.265122 23285 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.265633 22785 ts_manager.cc:194] Registered new tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.266254 22864 ts_manager.cc:194] Registered new tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.267431 23286 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.267601 23286 heartbeater.cc:461] Registering TS with master...
00:20:09.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.268425 23286 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:09.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.269420 22691 ts_manager.cc:194] Registered new tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:09.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.270319 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.66:53067
00:20:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.633711 23293 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.634056 23293 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.634135 23293 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.642838 23293 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.643006 23293 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.67
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.653646 23293 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:0
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.655334 23293 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:09.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.657428 23293 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.668454 23304 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.669123 23307 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:09.668478 23305 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.670449 23293 server_base.cc:1061] running on GCE node
00:20:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.671103 23293 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.672075 23293 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.673362 23293 hybrid_clock.cc:648] HybridClock initialized: now 1786494009673279 us; error 63 us; skew 500 ppm
00:20:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.673751 23293 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.676095 23293 webserver.cc:533] Webserver started at http://127.22.28.67:38727/ using document root <none> and password file <none>
00:20:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.677026 23293 fs_manager.cc:362] Metadata directory not provided
00:20:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.677170 23293 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.677597 23293 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:20:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.680298 23293 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/instance:
00:20:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.681149 23293 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal/instance:
00:20:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.687264 23293 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.001s
00:20:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.691125 23315 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.692973 23293 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
00:20:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.693164 23293 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.693415 23293 fs_report.cc:389] FS layout report
00:20:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/data
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.695430 23126 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.711833 23293 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:09.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.712852 23293 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:09.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.713121 23293 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.714079 23293 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.715988 23293 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:20:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.716078 23293 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:20:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.716204 23293 ts_tablet_manager.cc:616] Registered 0 tablets
00:20:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.716269 23293 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:20:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.761247 23293 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.67:41901
00:20:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.761355 23449 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.67:41901 every 8 connection(s)
00:20:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.762938 23293 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.765014 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 23293
00:20:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.765210 22641 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal/instance
00:20:09.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.799901 23452 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:09.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.800240 23451 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:09.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.800444 23451 heartbeater.cc:461] Registering TS with master...
00:20:09.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.800452 23452 heartbeater.cc:461] Registering TS with master...
00:20:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.801561 23451 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.801590 23452 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.803725 22785 ts_manager.cc:194] Registered new tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.803738 22691 ts_manager.cc:194] Registered new tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.804916 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.67:45639
00:20:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.807753 23450 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.807963 23450 heartbeater.cc:461] Registering TS with master...
00:20:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.808467 23450 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.809616 22864 ts_manager.cc:194] Registered new tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:09.823673 22641 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
00:20:09.825 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:20:09.827 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

00:20:09.836 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.22.28.126"
      port: 40901
    }
    bound_http_address {
      host: "127.22.28.126"
      port: 41549
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.22.28.125"
      port: 43953
    }
    bound_http_address {
      host: "127.22.28.125"
      port: 44931
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.22.28.124"
      port: 44295
    }
    bound_http_address {
      host: "127.22.28.124"
      port: 37597
    }
  }
}

00:20:09.839 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

00:20:09.842 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.22.28.65"
      port: 33753
    }
    bound_http_address {
      host: "127.22.28.65"
      port: 44085
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.22.28.66"
      port: 40559
    }
    bound_http_address {
      host: "127.22.28.66"
      port: 41155
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.22.28.67"
      port: 41901
    }
    bound_http_address {
      host: "127.22.28.67"
      port: 38727
    }
  }
}

00:20:09.843 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:20:10.039 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
00:20:10.097 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
00:20:10.263 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.28.125:43953(127.22.28.125:43953)] connecting to peer
00:20:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.273448 23286 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:10.328 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.28.126:40901(127.22.28.126:40901)] connecting to peer
00:20:10.330 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.28.124:44295(127.22.28.124:44295)] connecting to peer
00:20:10.348 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.28.126:40901(127.22.28.126:40901)] Successfully connected to peer
00:20:10.352 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.28.125:43953(127.22.28.125:43953)] Successfully connected to peer
00:20:10.352 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.22.28.124:44295(127.22.28.124:44295)] Successfully connected to peer
00:20:10.458 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.22.28.125
00:20:10.458 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.22.28.126
00:20:10.461 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.22.28.124
00:20:10.691 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x43d5ee9f, L:/127.0.0.1:55834 - R:/127.22.28.125:43953] using SASL/PLAIN
00:20:10.692 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x18fc3ca5, L:/127.0.0.1:51720 - R:/127.22.28.126:40901] using SASL/PLAIN
00:20:10.692 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xfadf6153, L:/127.0.0.1:42054 - R:/127.22.28.124:44295] using SASL/PLAIN
00:20:10.729 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
00:20:10.730 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=510), Traces: [0ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.28.125:43953, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.28.126:40901, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.28.124:44295, [483ms] Sub RPC ConnectToMaster: received response from server master-127.22.28.124:44295: OK, [483ms] Sub RPC ConnectToMaster: received response from server master-127.22.28.125:43953: OK, [484ms] Sub RPC ConnectToMaster: received response from server master-127.22.28.126:40901: OK, deferred=null) after lookup
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.783573 22691 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51720:
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205"
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:20:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:20:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:20:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
00:20:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
00:20:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
00:20:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:20:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:20:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:10.785980 22691 catalog_manager.cc:7099] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
00:20:10.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.808342 23451 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:10.836 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 5caefed0c8884e28a9c1c0bb9c5c16f0
00:20:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.849653 23213 tablet_service.cc:1511] Processing CreateTablet for tablet dffc135da8c24a56a4d38372feab7346 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205 [id=5caefed0c8884e28a9c1c0bb9c5c16f0]), partition=RANGE (key) PARTITION UNBOUNDED
00:20:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.851934 23213 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dffc135da8c24a56a4d38372feab7346. 1 dirs total, 0 dirs full, 0 dirs failed
00:20:10.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.862649 23051 tablet_service.cc:1511] Processing CreateTablet for tablet dffc135da8c24a56a4d38372feab7346 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205 [id=5caefed0c8884e28a9c1c0bb9c5c16f0]), partition=RANGE (key) PARTITION UNBOUNDED
00:20:10.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.864063 23369 tablet_service.cc:1511] Processing CreateTablet for tablet dffc135da8c24a56a4d38372feab7346 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205 [id=5caefed0c8884e28a9c1c0bb9c5c16f0]), partition=RANGE (key) PARTITION UNBOUNDED
00:20:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.865949 23051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dffc135da8c24a56a4d38372feab7346. 1 dirs total, 0 dirs full, 0 dirs failed
00:20:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.865981 23369 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dffc135da8c24a56a4d38372feab7346. 1 dirs total, 0 dirs full, 0 dirs failed
00:20:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.868001 23490 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap starting.
00:20:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.873808 23490 tablet_bootstrap.cc:654] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Neither blocks nor log segments found. Creating new log.
00:20:10.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.875710 23490 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Log is configured to *not* fsync() on all Append() calls
00:20:10.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.877899 23493 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap starting.
00:20:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.879123 23490 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: No bootstrap required, opened a new log
00:20:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.879705 23490 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent bootstrapping tablet: real 0.012s	user 0.009s	sys 0.000s
00:20:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.879765 23494 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap starting.
00:20:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.882761 23493 tablet_bootstrap.cc:654] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Neither blocks nor log segments found. Creating new log.
00:20:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.884277 23494 tablet_bootstrap.cc:654] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Neither blocks nor log segments found. Creating new log.
00:20:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.884624 23493 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Log is configured to *not* fsync() on all Append() calls
00:20:10.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.885913 23494 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Log is configured to *not* fsync() on all Append() calls
00:20:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.887485 23490 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.887908 23490 raft_consensus.cc:385] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:20:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.888021 23490 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cfbe5cfb9f79473db877ca144fd09767, State: Initialized, Role: FOLLOWER
00:20:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.888695 23493 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: No bootstrap required, opened a new log
00:20:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.888957 23494 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: No bootstrap required, opened a new log
00:20:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.888849 23490 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.889050 23493 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent bootstrapping tablet: real 0.011s	user 0.005s	sys 0.004s
00:20:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.889243 23494 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent bootstrapping tablet: real 0.010s	user 0.005s	sys 0.003s
00:20:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.897225 23494 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.898615 23494 raft_consensus.cc:385] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:20:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.899045 23494 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Initialized, Role: FOLLOWER
00:20:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.900389 23494 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:10.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.910013 23494 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent starting tablet: real 0.021s	user 0.008s	sys 0.007s
00:20:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.910308 23493 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:10.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.911967 23493 raft_consensus.cc:385] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:20:10.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.912356 23493 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Initialized, Role: FOLLOWER
00:20:10.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.913535 23493 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.919745 23490 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent starting tablet: real 0.040s	user 0.015s	sys 0.016s
00:20:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:10.922415 23493 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent starting tablet: real 0.033s	user 0.014s	sys 0.002s
00:20:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:10.981882 23288 tablet.cc:2406] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:20:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:11.019091 23453 tablet.cc:2406] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:20:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:11.156935 23128 tablet.cc:2406] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:20:11.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.228046 23498 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:20:11.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.228490 23498 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.230509 23498 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.240334 23237 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767" is_pre_election: true
00:20:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.240617 23074 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" is_pre_election: true
00:20:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.240993 23237 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 0.
00:20:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.241165 23074 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 0.
00:20:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.241952 23316 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 
00:20:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.242489 23498 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Leader pre-election won for term 1
00:20:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.242723 23498 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:20:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.242837 23498 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 0 FOLLOWER]: Advancing to term 1
00:20:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.245350 23498 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:11.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.246042 23498 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 1 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.246811 23237 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.246811 23074 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.247047 23074 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 0 FOLLOWER]: Advancing to term 1
00:20:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.247047 23237 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 0 FOLLOWER]: Advancing to term 1
00:20:11.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.250469 23237 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 1.
00:20:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.250469 23074 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 1.
00:20:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.251128 23316 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 
00:20:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.251561 23498 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 1 FOLLOWER]: Leader election won for term 1
00:20:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.252182 23498 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 1 LEADER]: Becoming Leader. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Running, Role: LEADER
00:20:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.252751 23498 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:11.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.259498 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 reported cstate change: term changed from 0 to 1, leader changed from <none> to c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67). New cstate: current_term: 1 leader_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: UNKNOWN } } }
00:20:11.329 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 5caefed0c8884e28a9c1c0bb9c5c16f0
00:20:11.351 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:11.357 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:11.359 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [11ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=null) after lookup
00:20:11.368 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:11.371 [DEBUG - kudu-nio-3] (Connection.java:757) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] Successfully connected to peer
00:20:11.450 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x1ea24093, L:/127.0.0.1:48026 - R:/127.22.28.67:41901] using TOKEN/null
00:20:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.463148 23237 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 1 FOLLOWER]: Refusing update from remote peer c2f0ab76e269428f9c0201c5fbe88e37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:20:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.463150 23074 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 1 FOLLOWER]: Refusing update from remote peer c2f0ab76e269428f9c0201c5fbe88e37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:20:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.464612 23502 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:20:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.465176 23498 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:20:11.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.480230 23508 mvcc.cc:204] Tried to move back new op lower bound from 7317479470938415104 to 7317479470096248832. Current Snapshot: MvccSnapshot[applied={T|T < 7317479470938415104}]
00:20:11.607 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:11.611 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.67:41901
00:20:11.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.613859 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 23293
00:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.640775 23444 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:11.767987 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 23293
00:20:11.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:11.801 [DEBUG - kudu-nio-3] (Connection.java:239) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] handling channelInactive
00:20:11.802 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state READY due to: connection closed
00:20:11.802 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: connection closed
00:20:11.824 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:11.825 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:11.825 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [5ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@376175860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@42375a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73a60e83 -> wakeup thread Time-limited test)) after lookup
00:20:11.826 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:11.829 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.829 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.829 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.845 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:11.845 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:11.846 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [5ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [29ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [32ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [32ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [47ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@376175860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@42375a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73a60e83 -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899))) after lookup
00:20:11.847 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:11.849 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.849 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.850 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.885 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:11.885 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:11.886 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [5ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [29ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [32ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [32ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [47ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [49ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [53ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [53ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [87ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@376175860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@42375a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73a60e83 -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530))) after lookup
00:20:11.887 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:11.889 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.889 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:11.889 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.004 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:12.005 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:12.005 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=207), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [5ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [29ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [32ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [32ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [47ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [49ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [53ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [53ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [87ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [89ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [92ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [92ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [207ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@376175860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@42375a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73a60e83 -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069))) after lookup
00:20:12.006 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:12.007 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.008 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.008 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.345 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:12.345 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:12.346 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=548), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [5ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [29ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [32ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [32ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [47ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [49ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [53ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [53ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [87ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [89ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [92ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [92ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [207ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [208ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [211ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [211ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [544ms] refreshing cache from master, [544ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [548ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@376175860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@42375a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069) -> (continuation of Deferred@1300169287 after retry RPC@1678249610), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73a60e83 -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069) -> (continuation of Deferred@1300169287 after retry RPC@1678249610))) after lookup
00:20:12.346 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:12.348 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.348 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.348 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.424 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:12.425 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:12.425 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=628), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [5ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [29ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [32ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [32ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [47ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [49ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [53ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [53ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [87ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [89ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [92ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [92ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [207ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [208ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [211ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [211ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [544ms] refreshing cache from master, [544ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [548ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [549ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [551ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [551ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [623ms] refreshing cache from master, [623ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [627ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@376175860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@42375a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069) -> (continuation of Deferred@1300169287 after retry RPC@1678249610) -> (continuation of Deferred@1125005022 after retry RPC@1742085227), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73a60e83 -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069) -> (continuation of Deferred@1300169287 after retry RPC@1678249610) -> (continuation of Deferred@1125005022 after retry RPC@1742085227))) after lookup
00:20:12.426 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:12.427 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:12.427 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.041491 23533 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.041765 23533 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.043565 23533 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:13.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:13.046491 23159 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:13.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:13.048578 23159 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:13.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.052469 23074 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" is_pre_election: true
00:20:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.053311 23162 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37
00:20:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.053819 23533 raft_consensus.cc:2749] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
00:20:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.054703 23540 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.054957 23540 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.056805 23540 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:13.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:13.058698 22993 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:13.060736 22993 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.065011 23237 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767" is_pre_election: true
00:20:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.065238 23237 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 1.
00:20:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.065788 22992 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.066251 23540 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 1 FOLLOWER]: Leader pre-election won for term 2
00:20:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.066416 23540 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 1 FOLLOWER]: Starting leader election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.066537 23540 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 1 FOLLOWER]: Advancing to term 2
00:20:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.068790 23540 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.069526 23540 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 2 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:13.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.070304 23237 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:13.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.070537 23237 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 1 FOLLOWER]: Advancing to term 2
00:20:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:13.071308 22993 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.072927 23237 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 2.
00:20:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.073520 22992 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:13.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.073932 23540 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 2 FOLLOWER]: Leader election won for term 2
00:20:13.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.074440 23540 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 2 LEADER]: Becoming Leader. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Running, Role: LEADER
00:20:13.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.074977 23540 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.079368 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc reported cstate change: term changed from 1 to 2, leader changed from c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67) to 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65). New cstate: current_term: 2 leader_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: HEALTHY } } }
00:20:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:13.677152 22993 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc -> Peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Couldn't send request to peer c2f0ab76e269428f9c0201c5fbe88e37. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:13.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.679180 23237 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 2 FOLLOWER]: Refusing update from remote peer 48b11ebe2f3b40e98ce801db2d63ffdc: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
00:20:13.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:13.679983 23560 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Connected to new peer: Peer: permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
00:20:14.664 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:14.665 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:14.666 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2868), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [5ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [29ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [32ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [32ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [47ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [49ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [53ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [53ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [87ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [89ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [92ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [92ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [203ms] refreshing cache from master, [204ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [207ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [208ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [211ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [211ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [544ms] refreshing cache from master, [544ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [548ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [549ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [551ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [551ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [623ms] refreshing cache from master, [623ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [627ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [629ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [630ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [630ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2864ms] refreshing cache from master, [2864ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [2867ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@376175860(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@42375a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069) -> (continuation of Deferred@1300169287 after retry RPC@1678249610) -> (continuation of Deferred@1125005022 after retry RPC@1742085227) -> (continuation of Deferred@1299549717 after retry RPC@297197834), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@73a60e83 -> wakeup thread Time-limited test -> (continuation of Deferred@999873552 after retry RPC@893282899) -> (continuation of Deferred@1463100721 after retry RPC@1942258530) -> (continuation of Deferred@238124233 after retry RPC@1788111069) -> (continuation of Deferred@1300169287 after retry RPC@1678249610) -> (continuation of Deferred@1125005022 after retry RPC@1742085227) -> (continuation of Deferred@1299549717 after retry RPC@297197834))) after lookup
00:20:14.666 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:14.667 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] Successfully connected to peer
00:20:14.699 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xe47857bf, L:/127.0.0.1:38472 - R:/127.22.28.65:33753] using TOKEN/null
00:20:14.731 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.67:41901
00:20:14.732 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:14.733799 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.112257 23583 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.112686 23583 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.112802 23583 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.121820 23583 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.122079 23583 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.67
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.132815 23583 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.135142 23583 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:15.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.137244 23583 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.149190 23588 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.149156 23589 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:15.150454 23591 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.150655 23583 server_base.cc:1061] running on GCE node
00:20:15.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.151436 23583 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.152647 23583 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.153900 23583 hybrid_clock.cc:648] HybridClock initialized: now 1786494015153827 us; error 69 us; skew 500 ppm
00:20:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.154294 23583 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.156988 23583 webserver.cc:533] Webserver started at http://127.22.28.67:38727/ using document root <none> and password file <none>
00:20:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.157876 23583 fs_manager.cc:362] Metadata directory not provided
00:20:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.158039 23583 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.164366 23583 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.000s
00:20:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.167735 23599 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.169361 23583 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.169535 23583 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.170360 23583 fs_report.cc:389] FS layout report
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/data
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:15.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.194183 23583 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.195142 23583 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.195423 23583 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.196568 23583 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:15.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.198899 23608 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:15.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.205660 23583 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.205796 23583 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:20:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.205928 23583 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.209282 23583 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.209396 23583 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
00:20:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.210018 23608 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap starting.
00:20:15.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.236604 23608 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Log is configured to *not* fsync() on all Append() calls
00:20:15.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.260939 23608 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:15.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.261909 23608 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap complete.
00:20:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.264113 23608 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent bootstrapping tablet: real 0.055s	user 0.044s	sys 0.004s
00:20:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.269942 23583 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.67:41901
00:20:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.269989 23733 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.67:41901 every 8 connection(s)
00:20:15.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.272001 23583 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.274379 23608 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.275833 23608 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Initialized, Role: FOLLOWER
00:20:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.276890 23608 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.279708 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 23583
00:20:15.280 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.324046 23608 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent starting tablet: real 0.060s	user 0.020s	sys 0.036s
00:20:15.325 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:15.326 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.65:33753
00:20:15.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:20:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.328022 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 22975
00:20:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.344913 23735 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.345327 23735 heartbeater.cc:461] Registering TS with master...
00:20:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.346642 23735 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.348570 23734 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.348760 23734 heartbeater.cc:461] Registering TS with master...
00:20:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.349414 23734 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.350711 22691 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.350930 23736 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.351058 23736 heartbeater.cc:461] Registering TS with master...
00:20:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.351495 22864 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.351943 23736 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.352411 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.67:60255
00:20:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.353979 22784 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.354434 23120 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.359161 23735 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:15.459399 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 22975
00:20:15.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:15.488 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] handling channelInactive
00:20:15.489 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state READY due to: connection closed
00:20:15.502 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:15.503 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.504 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.504 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.524 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:15.525 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:15.525 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=3, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [15ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [15ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@876490426(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@12263786 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4198de0b -> wakeup thread Time-limited test)) after lookup
00:20:15.526 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:15.527 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.527 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.527 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.545 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:15.546 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:15.547 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=5, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [15ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [15ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [36ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [38ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@876490426(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@12263786 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4198de0b -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141))) after lookup
00:20:15.547 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:15.549 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.549 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.549 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.604 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:15.605 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:15.605 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=7, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=116), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [15ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [15ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [36ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [38ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [58ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [60ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [60ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [115ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@876490426(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@12263786 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4198de0b -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104))) after lookup
00:20:15.606 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:15.607 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.607 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.607 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.805 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:15.805 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:15.806 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=9, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=316), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [15ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [15ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [36ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [38ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [58ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [60ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [60ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [115ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [116ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [118ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [118ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [311ms] refreshing cache from master, [311ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [315ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@876490426(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@12263786 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104) -> (continuation of Deferred@2034024710 after retry RPC@754308948), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4198de0b -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104) -> (continuation of Deferred@2034024710 after retry RPC@754308948))) after lookup
00:20:15.806 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:15.808 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.809 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:15.809 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:16.525 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:16.525 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:16.526 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=11, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1037), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [15ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [15ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [36ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [38ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [58ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [60ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [60ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [115ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [116ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [118ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [118ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [311ms] refreshing cache from master, [311ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [315ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [317ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [320ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [320ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1031ms] refreshing cache from master, [1032ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [1035ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@876490426(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@12263786 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104) -> (continuation of Deferred@2034024710 after retry RPC@754308948) -> (continuation of Deferred@390096868 after retry RPC@2031508009), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4198de0b -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104) -> (continuation of Deferred@2034024710 after retry RPC@754308948) -> (continuation of Deferred@390096868 after retry RPC@2031508009))) after lookup
00:20:16.527 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:16.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:16.528 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:16.528 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.814383 23752 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 48b11ebe2f3b40e98ce801db2d63ffdc)
00:20:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.814597 23752 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.815455 23752 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:16.818490 23162 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.825265 23685 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" is_pre_election: true
00:20:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.825752 23685 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cfbe5cfb9f79473db877ca144fd09767 in term 1.
00:20:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.826785 23159 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc
00:20:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.827206 23752 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 2 FOLLOWER]: Leader pre-election won for term 3
00:20:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.827344 23752 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 48b11ebe2f3b40e98ce801db2d63ffdc)
00:20:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.827420 23752 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 2 FOLLOWER]: Advancing to term 3
00:20:16.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.829793 23752 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:16.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.830525 23752 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 3 election: Requested vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.831084 23685 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.831319 23685 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 1 FOLLOWER]: Advancing to term 3
00:20:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:16.832523 23162 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:16.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.835777 23685 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cfbe5cfb9f79473db877ca144fd09767 in term 3.
00:20:16.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.836431 23159 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc
00:20:16.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.836901 23752 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 3 FOLLOWER]: Leader election won for term 3
00:20:16.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.837389 23752 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 3 LEADER]: Becoming Leader. State: Replica: cfbe5cfb9f79473db877ca144fd09767, State: Running, Role: LEADER
00:20:16.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.838227 23752 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:16.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:16.842658 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 reported cstate change: term changed from 2 to 3, leader changed from 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65) to cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66). New cstate: current_term: 3 leader_uuid: "cfbe5cfb9f79473db877ca144fd09767" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: UNKNOWN } } }
00:20:17.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:17.267328 23162 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 -> Peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Couldn't send request to peer 48b11ebe2f3b40e98ce801db2d63ffdc. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:17.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:17.366894 23685 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 3 FOLLOWER]: Refusing update from remote peer cfbe5cfb9f79473db877ca144fd09767: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
00:20:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:17.367888 23752 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
00:20:18.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:18.261080 23162 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111) [suppressed 4 similar messages]
00:20:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:19.633018 23162 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 -> Peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Couldn't send request to peer 48b11ebe2f3b40e98ce801db2d63ffdc. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:20:20.045 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:20.045 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:20.046 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=13, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=4556), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [0ms] delaying RPC due to: Illegal state: connection is terminated, [12ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [15ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [15ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [36ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [38ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [58ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [60ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [60ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [115ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [116ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [118ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [118ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [311ms] refreshing cache from master, [311ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [315ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [317ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [320ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [320ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1031ms] refreshing cache from master, [1032ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [1035ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [1037ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1039ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1039ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [4552ms] refreshing cache from master, [4552ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [4555ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@876490426(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@12263786 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104) -> (continuation of Deferred@2034024710 after retry RPC@754308948) -> (continuation of Deferred@390096868 after retry RPC@2031508009) -> (continuation of Deferred@105363360 after retry RPC@1305174611), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4198de0b -> wakeup thread Time-limited test -> (continuation of Deferred@575247097 after retry RPC@623278141) -> (continuation of Deferred@2068213157 after retry RPC@1840715104) -> (continuation of Deferred@2034024710 after retry RPC@754308948) -> (continuation of Deferred@390096868 after retry RPC@2031508009) -> (continuation of Deferred@105363360 after retry RPC@1305174611))) after lookup
00:20:20.047 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:20.048 [DEBUG - kudu-nio-1] (Connection.java:757) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] Successfully connected to peer
00:20:20.066 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x52249033, L:/127.0.0.1:35348 - R:/127.22.28.66:40559] using TOKEN/null
00:20:20.110 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.65:33753
00:20:20.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.112311 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:33753
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44085
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.502830 23794 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.503299 23794 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.503394 23794 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:20.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.512210 23794 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:20.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.512403 23794 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.65
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.523056 23794 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:33753
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44085
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.525261 23794 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.527558 23794 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.539667 23801 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.540321 23802 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.541194 23804 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.546000 23794 server_base.cc:1061] running on GCE node
00:20:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.546828 23794 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:20.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.548067 23794 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:20.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.549346 23794 hybrid_clock.cc:648] HybridClock initialized: now 1786494020549263 us; error 67 us; skew 500 ppm
00:20:20.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.549764 23794 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.552757 23794 webserver.cc:533] Webserver started at http://127.22.28.65:44085/ using document root <none> and password file <none>
00:20:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.553567 23794 fs_manager.cc:362] Metadata directory not provided
00:20:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.553689 23794 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:20.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.560041 23794 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.001s
00:20:20.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.563508 23811 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:20.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.565284 23794 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
00:20:20.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.565539 23794 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:20.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:20.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:08 on dist-test-slave-8w9v"
00:20:20.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.566479 23794 fs_report.cc:389] FS layout report
00:20:20.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/data
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.604686 23794 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.605784 23794 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.606174 23794 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.607394 23794 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.610764 23820 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.617854 23794 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.617983 23794 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
00:20:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.618103 23794 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.621627 23794 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.621742 23794 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
00:20:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.622364 23820 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap starting.
00:20:20.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.651357 23820 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Log is configured to *not* fsync() on all Append() calls
00:20:20.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.667325 23794 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.65:33753
00:20:20.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.667392 23941 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.65:33753 every 8 connection(s)
00:20:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.668944 23794 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:20.672 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.671715 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 23794
00:20:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.673475 23820 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.674218 23820 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap complete.
00:20:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.676308 23820 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent bootstrapping tablet: real 0.054s	user 0.045s	sys 0.008s
00:20:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.685392 23820 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.686849 23820 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Initialized, Role: FOLLOWER
00:20:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.689023 23820 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:20.700 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:20.703 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.66:40559
00:20:20.704 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:20:20.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.704808 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 23136
00:20:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.733705 23820 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent starting tablet: real 0.057s	user 0.022s	sys 0.017s
00:20:20.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.753479 23943 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:20.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.754073 23943 heartbeater.cc:461] Registering TS with master...
00:20:20.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.755839 23943 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.761229 22692 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.763557 22692 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.65:41223
00:20:20.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.765846 23942 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:20.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.766048 23942 heartbeater.cc:461] Registering TS with master...
00:20:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.766784 23942 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.768623 22863 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.769122 23943 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:20.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.772326 23280 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:20.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.774271 23944 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:20.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.774513 23944 heartbeater.cc:461] Registering TS with master...
00:20:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.774914 23944 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:20.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.777172 22784 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:20.886469 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 23136
00:20:20.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:20.925522 23958 negotiation.cc:336] Failed RPC negotiation. Trace:
00:20:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.885143 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.22.28.66:40875 (local address 127.22.28.65:33753)
00:20:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.885962 (+   819us) server_negotiation.cc:207] Beginning negotiation
00:20:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.885973 (+    11us) server_negotiation.cc:400] Waiting for connection header
00:20:20.925 [DEBUG - kudu-nio-1] (Connection.java:239) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] handling channelInactive
00:20:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.886395 (+   422us) server_negotiation.cc:408] Connection header received
00:20:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.886541 (+   146us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
00:20:20.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state READY due to: connection closed
00:20:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.886559 (+    18us) server_negotiation.cc:462] Received NEGOTIATE request from client
00:20:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.889713 (+  3154us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
00:20:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.889923 (+   210us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
00:20:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:20.924057 (+ 34134us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.22.28.66:40875: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:20:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":598,"thread_start_us":391,"threads_started":1}
00:20:20.926 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: connection closed
00:20:20.944 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:20.945 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:20.945 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1618679268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@65b519c8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ed4394e -> wakeup thread Time-limited test)) after lookup
00:20:20.946 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:20.947 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:20.947 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:20.947 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:20.964 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:20.964 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:20.965 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [20ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [22ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [22ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1618679268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@65b519c8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ed4394e -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605))) after lookup
00:20:20.965 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:20.966 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:20.967 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:20.967 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.004 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:21.004 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:21.005 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [20ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [22ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [22ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [40ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [42ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [42ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1618679268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@65b519c8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ed4394e -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907))) after lookup
00:20:21.005 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:21.007 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.007 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.007 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.145 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:21.145 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:21.145 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=220), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [20ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [22ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [22ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [40ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [42ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [42ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [80ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [82ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [82ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [219ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1618679268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@65b519c8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907) -> (continuation of Deferred@516731109 after retry RPC@1675696629), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ed4394e -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907) -> (continuation of Deferred@516731109 after retry RPC@1675696629))) after lookup
00:20:21.146 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:21.147 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.147 [DEBUG - kudu-nio-5] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.147 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.564 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:21.564 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:21.565 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=639), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [20ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [22ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [22ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [40ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [42ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [42ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [80ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [82ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [82ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [219ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [221ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [222ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [222ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [635ms] refreshing cache from master, [636ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [639ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1618679268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@65b519c8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907) -> (continuation of Deferred@516731109 after retry RPC@1675696629) -> (continuation of Deferred@744457891 after retry RPC@1192893548), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ed4394e -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907) -> (continuation of Deferred@516731109 after retry RPC@1675696629) -> (continuation of Deferred@744457891 after retry RPC@1192893548))) after lookup
00:20:21.565 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:21.566 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.567 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:21.567 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.135408 23961 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader cfbe5cfb9f79473db877ca144fd09767)
00:20:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.135735 23961 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.137696 23961 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:22.139076 23600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:22.141733 23600 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.147610 23889 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" is_pre_election: true
00:20:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.148190 23889 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 2.
00:20:22.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.149240 23605 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37; no voters: cfbe5cfb9f79473db877ca144fd09767
00:20:22.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.149713 23961 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 3 FOLLOWER]: Leader pre-election won for term 4
00:20:22.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.149878 23961 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 3 FOLLOWER]: Starting leader election (detected failure of leader cfbe5cfb9f79473db877ca144fd09767)
00:20:22.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.150017 23961 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 3 FOLLOWER]: Advancing to term 4
00:20:22.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.152279 23961 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.153090 23961 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 4 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.153918 23889 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.154191 23889 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 2 FOLLOWER]: Advancing to term 4
00:20:22.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:22.155105 23600 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.158701 23889 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 4.
00:20:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.159328 23605 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37; no voters: cfbe5cfb9f79473db877ca144fd09767
00:20:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.159695 23961 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Leader election won for term 4
00:20:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.160286 23961 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 LEADER]: Becoming Leader. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Running, Role: LEADER
00:20:22.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.160900 23961 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.165369 22692 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 reported cstate change: term changed from 3 to 4, leader changed from cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66) to c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67). New cstate: current_term: 4 leader_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: UNKNOWN } } }
00:20:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:22.653767 23600 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 -> Peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Couldn't send request to peer cfbe5cfb9f79473db877ca144fd09767. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.776993 23889 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 4 FOLLOWER]: Refusing update from remote peer c2f0ab76e269428f9c0201c5fbe88e37: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
00:20:22.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:22.778139 23961 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
00:20:23.184 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:23.184 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:23.185 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2259), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [20ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [22ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [22ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [40ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [42ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [42ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [80ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [82ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [82ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [219ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [221ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [222ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [222ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [635ms] refreshing cache from master, [636ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [639ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [640ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [642ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [642ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2255ms] refreshing cache from master, [2255ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [2258ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1618679268(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@65b519c8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907) -> (continuation of Deferred@516731109 after retry RPC@1675696629) -> (continuation of Deferred@744457891 after retry RPC@1192893548) -> (continuation of Deferred@820720155 after retry RPC@663073046), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1ed4394e -> wakeup thread Time-limited test -> (continuation of Deferred@58785666 after retry RPC@1017231605) -> (continuation of Deferred@1737623732 after retry RPC@1447391907) -> (continuation of Deferred@516731109 after retry RPC@1675696629) -> (continuation of Deferred@744457891 after retry RPC@1192893548) -> (continuation of Deferred@820720155 after retry RPC@663073046))) after lookup
00:20:23.185 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:23.187 [DEBUG - kudu-nio-7] (Connection.java:757) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] Successfully connected to peer
00:20:23.225 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x0a8f3973, L:/127.0.0.1:36484 - R:/127.22.28.67:41901] using TOKEN/null
00:20:23.256 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.66:40559
00:20:23.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.257727 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:40559
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41155
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.649734 23997 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.650133 23997 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.650223 23997 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.659268 23997 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.659467 23997 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.66
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.671031 23997 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:40559
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41155
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:23.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.673017 23997 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.675128 23997 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.687196 24010 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.687196 24007 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:23.687235 24003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.688109 23997 server_base.cc:1061] running on GCE node
00:20:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.688987 23997 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.690327 23997 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.691591 23997 hybrid_clock.cc:648] HybridClock initialized: now 1786494023691532 us; error 45 us; skew 500 ppm
00:20:23.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.692016 23997 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.694651 23997 webserver.cc:533] Webserver started at http://127.22.28.66:41155/ using document root <none> and password file <none>
00:20:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.695587 23997 fs_manager.cc:362] Metadata directory not provided
00:20:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.695762 23997 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.702032 23997 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
00:20:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.705405 24017 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.706991 23997 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.707223 23997 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.708106 23997 fs_report.cc:389] FS layout report
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/data
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:23.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.732963 23997 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:23.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.734113 23997 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:23.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.734562 23997 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.735805 23997 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:23.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.738399 24027 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.745500 23997 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.745689 23997 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
00:20:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.745877 23997 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.751307 23997 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.751466 23997 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.006s	sys 0.000s
00:20:23.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.752727 24027 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap starting.
00:20:23.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.789268 24027 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Log is configured to *not* fsync() on all Append() calls
00:20:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.811283 23997 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.66:40559
00:20:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.811386 24156 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.66:40559 every 8 connection(s)
00:20:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.813414 23997 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:23.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.823463 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 23997
00:20:23.824 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:23.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.828343 24027 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.829342 24027 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap complete.
00:20:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.831647 24027 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent bootstrapping tablet: real 0.080s	user 0.047s	sys 0.023s
00:20:23.840 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.842370 24027 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.844494 24027 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: cfbe5cfb9f79473db877ca144fd09767, State: Initialized, Role: FOLLOWER
00:20:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.846385 24027 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.864509 24027 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent starting tablet: real 0.033s	user 0.012s	sys 0.017s
00:20:23.868 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.67:41901
00:20:23.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.869253 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 23583
00:20:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.869196 24161 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.869683 24161 heartbeater.cc:461] Registering TS with master...
00:20:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.871043 24161 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:23.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.875021 22785 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.889412 24157 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.889588 24157 heartbeater.cc:461] Registering TS with master...
00:20:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.890169 24157 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.892074 24159 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.892268 24159 heartbeater.cc:461] Registering TS with master...
00:20:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.892283 23729 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.892916 24159 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.894411 22692 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.894508 22863 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.896214 22692 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.66:51861
00:20:23.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:23.899824 24159 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:24.002408 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 23583
00:20:24.030 [DEBUG - kudu-nio-7] (Connection.java:239) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] handling channelInactive
00:20:24.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:24.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state READY due to: connection closed
00:20:24.031 [DEBUG - Time-limited test] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:24.033 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.033 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.033 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.044 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:24.045 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:24.045 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1033454611(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@371e5fb7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22fb8357 -> wakeup thread Time-limited test)) after lookup
00:20:24.046 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:24.047 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.047 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.048 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.064 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:24.064 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:24.064 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [14ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [17ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1033454611(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@371e5fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22fb8357 -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496))) after lookup
00:20:24.065 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:24.066 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.066 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.066 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.084 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:24.084 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:24.084 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [14ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [17ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [35ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [35ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [52ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1033454611(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@371e5fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22fb8357 -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782))) after lookup
00:20:24.085 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:24.086 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.087 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.087 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.144 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:24.145 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:24.145 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=114), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [14ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [17ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [35ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [35ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [52ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [54ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [56ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [56ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [113ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1033454611(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@371e5fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782) -> (continuation of Deferred@1756772382 after retry RPC@35453983), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22fb8357 -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782) -> (continuation of Deferred@1756772382 after retry RPC@35453983))) after lookup
00:20:24.146 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:24.147 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.147 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.148 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.365 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:24.365 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:24.366 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=334), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [14ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [17ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [35ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [35ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [52ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [54ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [56ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [56ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [113ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [114ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [117ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [117ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [330ms] refreshing cache from master, [330ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [333ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1033454611(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@371e5fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782) -> (continuation of Deferred@1756772382 after retry RPC@35453983) -> (continuation of Deferred@1933070612 after retry RPC@371304296), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22fb8357 -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782) -> (continuation of Deferred@1756772382 after retry RPC@35453983) -> (continuation of Deferred@1933070612 after retry RPC@371304296))) after lookup
00:20:24.367 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:24.368 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.368 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:24.368 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:25.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.191320 24175 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 4 FOLLOWER]: Starting pre-election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:25.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.191581 24175 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:25.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.193276 24175 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:25.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:25.194622 23815 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:25.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:25.201431 23815 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.202956 24101 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767" is_pre_election: true
00:20:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.203428 24101 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 3.
00:20:25.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.204555 23813 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:25.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.205020 24175 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 4 FOLLOWER]: Leader pre-election won for term 5
00:20:25.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.205178 24175 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 4 FOLLOWER]: Starting leader election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:25.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.205298 24175 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 4 FOLLOWER]: Advancing to term 5
00:20:25.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.207557 24175 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.208367 24175 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 5 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.209095 24101 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.209376 24101 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 3 FOLLOWER]: Advancing to term 5
00:20:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:25.210273 23815 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.213774 24101 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 5.
00:20:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.214375 23813 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.214811 24175 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 5 FOLLOWER]: Leader election won for term 5
00:20:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.215272 24175 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 5 LEADER]: Becoming Leader. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Running, Role: LEADER
00:20:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.215869 24175 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:25.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.220075 22692 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc reported cstate change: term changed from 4 to 5, leader changed from c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67) to 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65). New cstate: current_term: 5 leader_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: HEALTHY } } }
00:20:25.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:25.776542 23815 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc -> Peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Couldn't send request to peer c2f0ab76e269428f9c0201c5fbe88e37. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:25.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.836287 24101 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 5 FOLLOWER]: Refusing update from remote peer 48b11ebe2f3b40e98ce801db2d63ffdc: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
00:20:25.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:25.837409 24197 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Connected to new peer: Peer: permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
00:20:26.045 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:26.045 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:26.045 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2014), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [14ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [17ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [35ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [35ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [52ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [54ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [56ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [56ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [109ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [113ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [114ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [117ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [117ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [330ms] refreshing cache from master, [330ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [333ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [335ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [337ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [337ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2009ms] refreshing cache from master, [2010ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [2013ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1033454611(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@371e5fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782) -> (continuation of Deferred@1756772382 after retry RPC@35453983) -> (continuation of Deferred@1933070612 after retry RPC@371304296) -> (continuation of Deferred@619250120 after retry RPC@1063839962), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22fb8357 -> wakeup thread Time-limited test -> (continuation of Deferred@1171774089 after retry RPC@1614463496) -> (continuation of Deferred@454316790 after retry RPC@755562782) -> (continuation of Deferred@1756772382 after retry RPC@35453983) -> (continuation of Deferred@1933070612 after retry RPC@371304296) -> (continuation of Deferred@619250120 after retry RPC@1063839962))) after lookup
00:20:26.046 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:26.047 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] Successfully connected to peer
00:20:26.082 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0xee2c4010, L:/127.0.0.1:53880 - R:/127.22.28.65:33753] using TOKEN/null
00:20:26.114 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.67:41901
00:20:26.114 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.115556 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:26.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:26.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.501652 24215 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:26.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.502050 24215 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:26.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.502169 24215 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:26.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.511159 24215 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:26.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.511406 24215 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.67
00:20:26.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.522542 24215 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.525014 24215 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:26.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.527354 24215 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.538527 24223 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:26.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.539147 24222 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:26.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.541306 24225 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:26.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.541525 24215 server_base.cc:1061] running on GCE node
00:20:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.542313 24215 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:26.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.543705 24215 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:26.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.544950 24215 hybrid_clock.cc:648] HybridClock initialized: now 1786494026544891 us; error 59 us; skew 500 ppm
00:20:26.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.545267 24215 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.547973 24215 webserver.cc:533] Webserver started at http://127.22.28.67:38727/ using document root <none> and password file <none>
00:20:26.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.548902 24215 fs_manager.cc:362] Metadata directory not provided
00:20:26.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.549021 24215 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:26.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.555188 24215 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
00:20:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.558499 24232 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.560037 24215 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.560278 24215 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.561098 24215 fs_report.cc:389] FS layout report
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/data
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.583047 24215 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.583992 24215 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.584477 24215 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.585570 24215 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.588922 24239 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.595639 24215 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.595773 24215 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:20:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.595893 24215 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:26.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.599283 24215 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:26.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.599401 24215 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
00:20:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.600025 24239 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap starting.
00:20:26.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.629290 24239 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Log is configured to *not* fsync() on all Append() calls
00:20:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.651240 24215 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.67:41901
00:20:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.651252 24378 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.67:41901 every 8 connection(s)
00:20:26.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.653357 24215 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:26.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.660251 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 24215
00:20:26.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.666982 24239 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:26.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.667779 24239 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap complete.
00:20:26.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.669588 24239 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent bootstrapping tablet: real 0.070s	user 0.046s	sys 0.019s
00:20:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.677266 24239 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.678359 24239 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Initialized, Role: FOLLOWER
00:20:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.679325 24239 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:26.699 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:26.701 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.65:33753
00:20:26.702 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:20:26.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.702907 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 23794
00:20:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.717929 24382 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.718087 24239 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent starting tablet: real 0.048s	user 0.024s	sys 0.024s
00:20:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.718526 24382 heartbeater.cc:461] Registering TS with master...
00:20:26.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.719611 24382 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.723758 22785 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.725033 24380 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.725226 24380 heartbeater.cc:461] Registering TS with master...
00:20:26.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.725955 24380 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:26.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.727505 22691 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.729245 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.67:45511
00:20:26.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.732722 24380 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:26.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.739522 24379 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:26.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.739676 24379 heartbeater.cc:461] Registering TS with master...
00:20:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.740017 24379 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.740965 23937 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.742342 22863 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:26.849260 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 23794
00:20:26.879 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] handling channelInactive
00:20:26.880 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state READY due to: connection closed
00:20:26.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:26.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:26.880656 24402 negotiation.cc:336] Failed RPC negotiation. Trace:
00:20:26.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:26.849855 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.22.28.65:49311 (local address 127.22.28.67:41901)
00:20:26.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:26.850557 (+   702us) server_negotiation.cc:207] Beginning negotiation
00:20:26.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:26.850570 (+    13us) server_negotiation.cc:400] Waiting for connection header
00:20:26.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:26.878963 (+ 28393us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.22.28.65:49311: BlockingRecv error: recv got EOF from 127.22.28.65:49311 (error 108)
00:20:26.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":488,"thread_start_us":2145,"threads_started":1}
00:20:26.881 [DEBUG - Time-limited test] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:26.882 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.882 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.882 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.904 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:26.905 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:26.905 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [23ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@2002652007(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@f08136e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3704f717 -> wakeup thread Time-limited test)) after lookup
00:20:26.905 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:26.906 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.907 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.924 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:26.924 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:26.925 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [23ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [24ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [26ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [26ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@2002652007(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@f08136e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3704f717 -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205))) after lookup
00:20:26.925 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:26.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.926 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.964 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:26.965 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:26.965 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [23ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [24ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [26ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [26ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [44ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [45ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [45ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@2002652007(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@f08136e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3704f717 -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747))) after lookup
00:20:26.965 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:26.966 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.967 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:26.967 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.004 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:27.004 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:27.005 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=124), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [23ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [24ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [26ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [26ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [44ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [45ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [45ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [84ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [86ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [86ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [123ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@2002652007(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@f08136e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3704f717 -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553))) after lookup
00:20:27.005 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:27.006 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.006 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.007 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.204 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:27.205 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:27.205 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=324), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [23ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [24ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [26ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [26ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [44ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [45ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [45ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [84ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [86ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [86ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [123ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [124ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [126ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [126ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [320ms] refreshing cache from master, [320ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [323ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@2002652007(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@f08136e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553) -> (continuation of Deferred@1932396773 after retry RPC@791528286), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3704f717 -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553) -> (continuation of Deferred@1932396773 after retry RPC@791528286))) after lookup
00:20:27.206 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:27.207 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.207 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.207 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.804 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:27.805 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:27.805 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=924), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [23ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [24ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [26ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [26ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [44ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [45ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [45ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [84ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [86ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [86ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [123ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [124ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [126ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [126ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [320ms] refreshing cache from master, [320ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [323ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [324ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [326ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [326ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [920ms] refreshing cache from master, [920ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [923ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@2002652007(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@f08136e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553) -> (continuation of Deferred@1932396773 after retry RPC@791528286) -> (continuation of Deferred@376608982 after retry RPC@1459916405), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3704f717 -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553) -> (continuation of Deferred@1932396773 after retry RPC@791528286) -> (continuation of Deferred@376608982 after retry RPC@1459916405))) after lookup
00:20:27.806 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:27.807 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.807 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:27.808 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.045991 24403 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:20:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.046298 24403 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.048013 24403 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:28.053462 24236 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:28.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:28.055652 24236 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:28.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.060029 24101 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767" is_pre_election: true
00:20:28.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.061372 24233 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767
00:20:28.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.061831 24403 raft_consensus.cc:2749] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
00:20:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.076196 24409 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 48b11ebe2f3b40e98ce801db2d63ffdc)
00:20:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.076525 24409 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.078487 24409 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:28.094905 24023 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:28.097251 24023 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:28.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.102468 24319 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" is_pre_election: true
00:20:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.102977 24319 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cfbe5cfb9f79473db877ca144fd09767 in term 4.
00:20:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.104270 24019 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc
00:20:28.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.104827 24409 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 5 FOLLOWER]: Leader pre-election won for term 6
00:20:28.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.104979 24409 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 48b11ebe2f3b40e98ce801db2d63ffdc)
00:20:28.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.105050 24409 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 5 FOLLOWER]: Advancing to term 6
00:20:28.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.107225 24409 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.107949 24409 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 6 election: Requested vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.108665 24319 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.108964 24319 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 4 FOLLOWER]: Advancing to term 6
00:20:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:28.110211 24023 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:28.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.113595 24319 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cfbe5cfb9f79473db877ca144fd09767 in term 6.
00:20:28.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.114368 24019 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc
00:20:28.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.114768 24409 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 6 FOLLOWER]: Leader election won for term 6
00:20:28.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.115446 24409 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 6 LEADER]: Becoming Leader. State: Replica: cfbe5cfb9f79473db877ca144fd09767, State: Running, Role: LEADER
00:20:28.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.116276 24409 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.121398 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 reported cstate change: term changed from 5 to 6, leader changed from 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65) to cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66). New cstate: current_term: 6 leader_uuid: "cfbe5cfb9f79473db877ca144fd09767" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: UNKNOWN } } }
00:20:28.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.519845 24319 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 6 FOLLOWER]: Refusing update from remote peer cfbe5cfb9f79473db877ca144fd09767: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
00:20:28.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:28.521147 24409 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
00:20:28.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:28.545710 24023 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 -> Peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Couldn't send request to peer 48b11ebe2f3b40e98ce801db2d63ffdc. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:30.705123 24023 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 -> Peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Couldn't send request to peer 48b11ebe2f3b40e98ce801db2d63ffdc. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:20:31.885 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:31.885 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:31.885 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=5004), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [1ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [1ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [23ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [24ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [26ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [26ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [43ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [44ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [45ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [45ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [84ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [86ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [86ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [123ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [124ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [126ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [126ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [320ms] refreshing cache from master, [320ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [323ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [324ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [326ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [326ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [920ms] refreshing cache from master, [920ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [923ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [925ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [927ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [927ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [5000ms] refreshing cache from master, [5000ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [5003ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@2002652007(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@f08136e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553) -> (continuation of Deferred@1932396773 after retry RPC@791528286) -> (continuation of Deferred@376608982 after retry RPC@1459916405) -> (continuation of Deferred@672609928 after retry RPC@1536389849), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3704f717 -> wakeup thread Time-limited test -> (continuation of Deferred@1238177860 after retry RPC@1499591205) -> (continuation of Deferred@731718636 after retry RPC@1385677747) -> (continuation of Deferred@1999579396 after retry RPC@466220553) -> (continuation of Deferred@1932396773 after retry RPC@791528286) -> (continuation of Deferred@376608982 after retry RPC@1459916405) -> (continuation of Deferred@672609928 after retry RPC@1536389849))) after lookup
00:20:31.886 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:31.888 [DEBUG - kudu-nio-6] (Connection.java:757) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] Successfully connected to peer
00:20:31.910 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0xbbda51ee, L:/127.0.0.1:42694 - R:/127.22.28.66:40559] using TOKEN/null
00:20:31.944 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.65:33753
00:20:31.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:31.947163 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:33753
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44085
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:32.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.350569 24454 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:32.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.350945 24454 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:32.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.351084 24454 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:32.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.362980 24454 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:32.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.363178 24454 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.65
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.380609 24454 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:33753
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44085
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:32.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.383096 24454 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.385677 24454 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.400085 24460 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.401363 24461 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:32.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:32.402406 24463 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:32.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.404286 24454 server_base.cc:1061] running on GCE node
00:20:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.405184 24454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:32.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.406417 24454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:32.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.407843 24454 hybrid_clock.cc:648] HybridClock initialized: now 1786494032407764 us; error 80 us; skew 500 ppm
00:20:32.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.408316 24454 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:32.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.411826 24454 webserver.cc:533] Webserver started at http://127.22.28.65:44085/ using document root <none> and password file <none>
00:20:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.413277 24454 fs_manager.cc:362] Metadata directory not provided
00:20:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.413440 24454 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:32.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.420662 24454 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
00:20:32.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.424422 24471 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:32.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.426012 24454 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.426249 24454 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:08 on dist-test-slave-8w9v"
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.427140 24454 fs_report.cc:389] FS layout report
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/data
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:32.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.442536 24454 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:32.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.443614 24454 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:32.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.444001 24454 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:32.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.445191 24454 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.447505 24479 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.454241 24454 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.454420 24454 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:20:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.454582 24454 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.458061 24454 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.458215 24454 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
00:20:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.458766 24479 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap starting.
00:20:32.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.487195 24479 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Log is configured to *not* fsync() on all Append() calls
00:20:32.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.512475 24454 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.65:33753
00:20:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.512526 24604 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.65:33753 every 8 connection(s)
00:20:32.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.514668 24454 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:32.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.516327 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 24454
00:20:32.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.533020 24479 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.538609 24479 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap complete.
00:20:32.540 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:32.541 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.66:40559
00:20:32.541 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:20:32.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.542098 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 23997
00:20:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.543407 24479 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent bootstrapping tablet: real 0.085s	user 0.052s	sys 0.013s
00:20:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.551481 24479 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.552863 24479 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Initialized, Role: FOLLOWER
00:20:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.554025 24479 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.561421 24607 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:32.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.561977 24607 heartbeater.cc:461] Registering TS with master...
00:20:32.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.563550 24607 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:32.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.567288 22785 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.571730 24606 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.571962 24606 heartbeater.cc:461] Registering TS with master...
00:20:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.572714 24606 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:32.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.574625 22691 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:32.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.577550 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.65:34127
00:20:32.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.586409 24605 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:32.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.586627 24605 heartbeater.cc:461] Registering TS with master...
00:20:32.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.587209 24605 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:32.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.588160 24606 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.589677 22863 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:32.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.592263 24479 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent starting tablet: real 0.049s	user 0.020s	sys 0.020s
00:20:32.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.596501 24150 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:32.701172 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 23997
00:20:32.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:32.727 [DEBUG - kudu-nio-6] (Connection.java:239) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] handling channelInactive
00:20:32.728 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state READY due to: connection closed
00:20:32.742 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:32.743 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.743 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.743 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.764 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:32.765 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:32.765 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=3, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [16ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [16ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@794334662(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6f9d2f4e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ba6ce04 -> wakeup thread Time-limited test)) after lookup
00:20:32.766 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:32.767 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.767 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.767 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.786 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:32.786 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:32.786 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=5, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [16ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [16ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [40ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [40ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [58ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@794334662(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6f9d2f4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ba6ce04 -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370))) after lookup
00:20:32.787 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:32.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.789 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.844 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:32.845 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:32.845 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=7, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=117), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [16ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [16ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [40ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [40ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [58ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [60ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [62ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [62ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [117ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@794334662(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6f9d2f4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ba6ce04 -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818))) after lookup
00:20:32.846 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:32.847 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.847 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.847 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.944 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:32.945 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:32.945 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=9, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=217), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [16ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [16ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [40ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [40ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [58ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [60ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [62ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [62ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [117ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [118ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [120ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [120ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [217ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@794334662(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6f9d2f4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818) -> (continuation of Deferred@2067341609 after retry RPC@342133129), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ba6ce04 -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818) -> (continuation of Deferred@2067341609 after retry RPC@342133129))) after lookup
00:20:32.946 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:32.947 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.948 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:32.948 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:33.105 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:33.105 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:33.106 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=11, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=378), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [16ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [16ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [40ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [40ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [58ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [60ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [62ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [62ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [117ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [118ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [120ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [120ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [217ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [218ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [221ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [221ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [377ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@794334662(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6f9d2f4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818) -> (continuation of Deferred@2067341609 after retry RPC@342133129) -> (continuation of Deferred@1993943445 after retry RPC@1187106310), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ba6ce04 -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818) -> (continuation of Deferred@2067341609 after retry RPC@342133129) -> (continuation of Deferred@1993943445 after retry RPC@1187106310))) after lookup
00:20:33.106 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:33.107 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:33.107 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:33.107 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:33.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.936347 24621 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader cfbe5cfb9f79473db877ca144fd09767)
00:20:33.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.936578 24621 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:33.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.937429 24621 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:33.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:33.938575 24233 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:33.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:33.941211 24233 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.947587 24550 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" is_pre_election: true
00:20:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.948221 24550 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 5.
00:20:33.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.949446 24236 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37; no voters: cfbe5cfb9f79473db877ca144fd09767
00:20:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.949899 24621 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 6 FOLLOWER]: Leader pre-election won for term 7
00:20:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.950062 24621 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 6 FOLLOWER]: Starting leader election (detected failure of leader cfbe5cfb9f79473db877ca144fd09767)
00:20:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.950155 24621 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 6 FOLLOWER]: Advancing to term 7
00:20:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.952986 24621 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.953756 24621 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 7 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:33.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.954646 24550 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:33.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.954947 24550 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 5 FOLLOWER]: Advancing to term 7
00:20:33.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:33.955734 24233 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:33.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.959328 24550 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 7.
00:20:33.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.960039 24236 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37; no voters: cfbe5cfb9f79473db877ca144fd09767
00:20:33.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.960474 24621 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Leader election won for term 7
00:20:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.961012 24621 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 LEADER]: Becoming Leader. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Running, Role: LEADER
00:20:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.961611 24621 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:33.965853 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 reported cstate change: term changed from 6 to 7, leader changed from cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66) to c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67). New cstate: current_term: 7 leader_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: UNKNOWN } } }
00:20:34.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:34.383818 24233 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 -> Peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Couldn't send request to peer cfbe5cfb9f79473db877ca144fd09767. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:34.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:34.389796 24550 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 7 FOLLOWER]: Refusing update from remote peer c2f0ab76e269428f9c0201c5fbe88e37: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
00:20:34.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:34.390818 24621 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
00:20:35.585 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:35.585 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:35.586 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=13, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2858), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [1ms] delaying RPC due to: Illegal state: connection is terminated, [14ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [16ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [16ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [40ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [40ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [58ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [60ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [62ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [62ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [117ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [118ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [120ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [120ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [217ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [218ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [221ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [221ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [377ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [379ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [380ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [380ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2854ms] refreshing cache from master, [2854ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [2857ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@794334662(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6f9d2f4e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818) -> (continuation of Deferred@2067341609 after retry RPC@342133129) -> (continuation of Deferred@1993943445 after retry RPC@1187106310) -> (continuation of Deferred@1237131616 after retry RPC@21260207), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ba6ce04 -> wakeup thread Time-limited test -> (continuation of Deferred@921229085 after retry RPC@1887579370) -> (continuation of Deferred@513144008 after retry RPC@253498818) -> (continuation of Deferred@2067341609 after retry RPC@342133129) -> (continuation of Deferred@1993943445 after retry RPC@1187106310) -> (continuation of Deferred@1237131616 after retry RPC@21260207))) after lookup
00:20:35.587 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:35.588 [DEBUG - kudu-nio-5] (Connection.java:757) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] Successfully connected to peer
00:20:35.621 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0x0dd7b95b, L:/127.0.0.1:33602 - R:/127.22.28.67:41901] using TOKEN/null
00:20:35.670 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.66:40559
00:20:35.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:20:35.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:35.671720 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:40559
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41155
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.063781 24666 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.064213 24666 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.064309 24666 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.073153 24666 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.073336 24666 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.66
00:20:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.084251 24666 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:40559
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41155
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:36.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.086570 24666 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:36.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.089334 24666 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:36.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.102306 24673 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:36.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.102432 24674 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.103891 24676 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.105167 24666 server_base.cc:1061] running on GCE node
00:20:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.105934 24666 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.107035 24666 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.108295 24666 hybrid_clock.cc:648] HybridClock initialized: now 1786494036108217 us; error 59 us; skew 500 ppm
00:20:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.108721 24666 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:36.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.111495 24666 webserver.cc:533] Webserver started at http://127.22.28.66:41155/ using document root <none> and password file <none>
00:20:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.112550 24666 fs_manager.cc:362] Metadata directory not provided
00:20:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.112716 24666 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:36.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.119519 24666 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
00:20:36.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.123004 24685 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.124778 24666 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.125016 24666 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.125972 24666 fs_report.cc:389] FS layout report
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/data
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:36.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.174794 24666 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:36.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.175776 24666 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:36.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.176100 24666 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.177352 24666 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.179695 24694 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.186560 24666 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.186689 24666 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:20:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.186806 24666 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:36.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.190376 24666 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:36.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.190502 24666 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.002s
00:20:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.191246 24694 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap starting.
00:20:36.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.220556 24694 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Log is configured to *not* fsync() on all Append() calls
00:20:36.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.245641 24666 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.66:40559
00:20:36.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.245703 24821 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.66:40559 every 8 connection(s)
00:20:36.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.247208 24666 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:36.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.248644 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 24666
00:20:36.249 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.260043 24694 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:36.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.265398 24694 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap complete.
00:20:36.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.267508 24694 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent bootstrapping tablet: real 0.077s	user 0.061s	sys 0.008s
00:20:36.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.279114 24694 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:36.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.280582 24694 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: cfbe5cfb9f79473db877ca144fd09767, State: Initialized, Role: FOLLOWER
00:20:36.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.282430 24694 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:36.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.290316 24826 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:36.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.290823 24826 heartbeater.cc:461] Registering TS with master...
00:20:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.292205 24826 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.292424 24694 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent starting tablet: real 0.025s	user 0.017s	sys 0.004s
00:20:36.292 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:36.293 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.67:41901
00:20:36.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:36.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.294116 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 24215
00:20:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.296452 22785 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:36.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.306679 24823 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:36.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.306917 24823 heartbeater.cc:461] Registering TS with master...
00:20:36.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.307566 24823 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:36.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.309564 22864 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:36.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.314251 24373 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.316298 24825 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.316461 24825 heartbeater.cc:461] Registering TS with master...
00:20:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.317000 24825 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:36.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.318862 22692 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:36.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.320188 22692 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.66:56747
00:20:36.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.322780 24825 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:36.422891 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 24215
00:20:36.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:36.451 [DEBUG - kudu-nio-5] (Connection.java:239) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] handling channelInactive
00:20:36.451 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state READY due to: connection closed
00:20:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.450340 22848 connection.cc:573] server connection from 127.22.28.67:54263 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:20:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.450829 22768 connection.cc:573] server connection from 127.22.28.67:36789 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:20:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:36.451534 22675 connection.cc:573] server connection from 127.22.28.67:45511 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:20:36.452 [DEBUG - Time-limited test] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:36.453 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.453 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.453 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.465 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:36.465 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:36.465 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@244883566(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@2cc3c70d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@120d4d08 -> wakeup thread Time-limited test)) after lookup
00:20:36.466 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:36.467 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.467 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.467 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.484 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:36.485 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:36.485 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [15ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [16ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@244883566(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@2cc3c70d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@120d4d08 -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548))) after lookup
00:20:36.485 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:36.486 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.487 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.487 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.524 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:36.524 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:36.525 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [15ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [16ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [36ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [36ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [73ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@244883566(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@2cc3c70d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@120d4d08 -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115))) after lookup
00:20:36.525 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:36.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.526 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.644 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:36.644 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:36.644 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=193), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [15ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [16ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [36ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [36ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [73ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [74ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [75ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [75ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [193ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@244883566(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@2cc3c70d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115) -> (continuation of Deferred@658029213 after retry RPC@307197891), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@120d4d08 -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115) -> (continuation of Deferred@658029213 after retry RPC@307197891))) after lookup
00:20:36.645 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:36.646 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.647 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.647 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.704 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:36.704 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:36.705 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=253), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [15ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [16ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [36ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [36ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [73ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [74ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [75ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [75ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [193ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [194ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [196ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [196ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [249ms] refreshing cache from master, [249ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [253ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@244883566(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@2cc3c70d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115) -> (continuation of Deferred@658029213 after retry RPC@307197891) -> (continuation of Deferred@172687401 after retry RPC@1666420684), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@120d4d08 -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115) -> (continuation of Deferred@658029213 after retry RPC@307197891) -> (continuation of Deferred@172687401 after retry RPC@1666420684))) after lookup
00:20:36.705 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:36.706 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.707 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:36.707 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:37.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.715013 24844 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 7 FOLLOWER]: Starting pre-election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:37.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.715235 24844 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.717057 24844 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:37.718386 24474 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:37.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:37.720283 24474 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:37.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.726469 24768 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767" is_pre_election: true
00:20:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.726945 24768 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 6.
00:20:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.727960 24473 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.728488 24844 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 7 FOLLOWER]: Leader pre-election won for term 8
00:20:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.728646 24844 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 7 FOLLOWER]: Starting leader election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.728778 24844 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 7 FOLLOWER]: Advancing to term 8
00:20:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.730945 24844 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.731868 24844 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 8 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.732553 24768 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.732825 24768 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 6 FOLLOWER]: Advancing to term 8
00:20:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:37.733722 24474 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:37.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.737141 24768 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 8.
00:20:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.737818 24473 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.738198 24844 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Leader election won for term 8
00:20:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.738695 24844 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 LEADER]: Becoming Leader. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Running, Role: LEADER
00:20:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.739249 24844 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.743711 22692 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc reported cstate change: term changed from 7 to 8, leader changed from c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67) to 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65). New cstate: current_term: 8 leader_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: HEALTHY } } }
00:20:37.804 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:37.804 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:37.805 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1353), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [2ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [13ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [15ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [16ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [16ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [33ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [34ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [36ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [36ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [73ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [74ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [75ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [75ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [193ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [194ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [196ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [196ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [249ms] refreshing cache from master, [249ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [253ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [254ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [256ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [256ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [1349ms] refreshing cache from master, [1349ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [1353ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@244883566(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@2cc3c70d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115) -> (continuation of Deferred@658029213 after retry RPC@307197891) -> (continuation of Deferred@172687401 after retry RPC@1666420684) -> (continuation of Deferred@1026153941 after retry RPC@1123752019), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@120d4d08 -> wakeup thread Time-limited test -> (continuation of Deferred@886182419 after retry RPC@1280273548) -> (continuation of Deferred@555752687 after retry RPC@373047115) -> (continuation of Deferred@658029213 after retry RPC@307197891) -> (continuation of Deferred@172687401 after retry RPC@1666420684) -> (continuation of Deferred@1026153941 after retry RPC@1123752019))) after lookup
00:20:37.806 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:37.807 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] Successfully connected to peer
00:20:37.841 [DEBUG - kudu-nio-4] (Negotiator.java:918) Authenticated connection [id: 0xfadac813, L:/127.0.0.1:56586 - R:/127.22.28.65:33753] using TOKEN/null
00:20:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.848637 24768 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 8 FOLLOWER]: Refusing update from remote peer 48b11ebe2f3b40e98ce801db2d63ffdc: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
00:20:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.849934 24849 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Connected to new peer: Peer: permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
00:20:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:37.850761 24474 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc -> Peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Couldn't send request to peer c2f0ab76e269428f9c0201c5fbe88e37. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:37.889 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.67:41901
00:20:37.890 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:37.890771 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.274274 24877 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.274669 24877 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.274763 24877 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.283830 24877 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.284024 24877 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.67
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.295666 24877 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.297662 24877 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.299636 24877 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.311566 24886 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.311602 24885 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.313048 24888 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.313197 24877 server_base.cc:1061] running on GCE node
00:20:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.314103 24877 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.315116 24877 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.316375 24877 hybrid_clock.cc:648] HybridClock initialized: now 1786494038316294 us; error 65 us; skew 500 ppm
00:20:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.316809 24877 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.319449 24877 webserver.cc:533] Webserver started at http://127.22.28.67:38727/ using document root <none> and password file <none>
00:20:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.320493 24877 fs_manager.cc:362] Metadata directory not provided
00:20:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.320653 24877 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:38.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.327301 24877 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
00:20:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.330964 24894 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.332861 24877 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.333048 24877 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.333976 24877 fs_report.cc:389] FS layout report
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/data
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.355484 24877 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.356600 24877 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.357014 24877 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:38.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.358295 24877 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:38.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.360951 24904 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:38.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.367604 24877 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.367733 24877 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:20:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.367869 24877 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.371275 24877 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.371392 24877 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
00:20:38.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.372159 24904 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap starting.
00:20:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.404313 24904 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Log is configured to *not* fsync() on all Append() calls
00:20:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.417832 24877 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.67:41901
00:20:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.417943 25033 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.67:41901 every 8 connection(s)
00:20:38.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.419502 24877 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:38.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.429215 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 24877
00:20:38.429 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:38.458 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:38.459 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.65:33753
00:20:38.460 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:20:38.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.460356 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 24454
00:20:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.487172 25034 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.487318 25035 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.487680 25035 heartbeater.cc:461] Registering TS with master...
00:20:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.487687 25034 heartbeater.cc:461] Registering TS with master...
00:20:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.489149 25034 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.489295 25036 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.489456 25036 heartbeater.cc:461] Registering TS with master...
00:20:38.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.490000 25036 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.492442 24600 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.493021 22863 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.493310 22785 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.493710 25035 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.495215 22691 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.496619 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.67:41365
00:20:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.503115 24904 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.504077 24904 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap complete.
00:20:38.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.505535 24904 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent bootstrapping tablet: real 0.134s	user 0.067s	sys 0.013s
00:20:38.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.509995 24904 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.510854 24904 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Initialized, Role: FOLLOWER
00:20:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.511658 24904 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.513805 24904 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent starting tablet: real 0.008s	user 0.003s	sys 0.005s
00:20:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.514107 25035 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:38.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:38.620285 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 24454
00:20:38.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:38.647223 24689 connection.cc:573] server connection from 127.22.28.65:59671 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:20:38.647 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] handling channelInactive
00:20:38.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:38.648 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state READY due to: connection closed
00:20:38.648 [DEBUG - Time-limited test] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:38.649 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.650 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.650 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.664 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:38.664 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:38.665 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [2ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [2ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@135170592(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@122d22f2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a24649a -> wakeup thread Time-limited test)) after lookup
00:20:38.665 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:38.666 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.666 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.667 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.684 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:38.684 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:38.684 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [2ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [2ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [17ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [19ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [19ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@135170592(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@122d22f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a24649a -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147))) after lookup
00:20:38.685 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:38.686 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.686 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.687 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.704 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:38.704 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:38.705 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [2ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [2ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [17ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [19ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [19ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [37ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [39ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@135170592(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@122d22f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a24649a -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742))) after lookup
00:20:38.706 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:38.707 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.707 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.707 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.724 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:38.725 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:38.725 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [2ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [2ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [17ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [19ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [19ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [37ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [39ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [57ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [59ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [59ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [76ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@135170592(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@122d22f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742) -> (continuation of Deferred@1909384575 after retry RPC@113347751), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a24649a -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742) -> (continuation of Deferred@1909384575 after retry RPC@113347751))) after lookup
00:20:38.726 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:38.727 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.727 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:38.727 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:39.103 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:39.104 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:39.104 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=456), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [2ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [2ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [17ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [19ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [19ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [37ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [39ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [57ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [59ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [59ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [76ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [77ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [79ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [79ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [452ms] refreshing cache from master, [452ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [455ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@135170592(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@122d22f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742) -> (continuation of Deferred@1909384575 after retry RPC@113347751) -> (continuation of Deferred@1455789927 after retry RPC@53601284), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a24649a -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742) -> (continuation of Deferred@1909384575 after retry RPC@113347751) -> (continuation of Deferred@1455789927 after retry RPC@53601284))) after lookup
00:20:39.105 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:39.106 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:39.106 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:39.106 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753
00:20:39.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:39.649765 25052 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:20:39.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:39.650086 25052 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:39.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:39.651773 25052 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:39.653694 24898 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:39.655529 24898 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:39.658869 24768 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767" is_pre_election: true
00:20:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:39.659792 24895 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767
00:20:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:39.660328 25052 raft_consensus.cc:2749] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
00:20:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.124389 25057 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 48b11ebe2f3b40e98ce801db2d63ffdc)
00:20:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.124636 25057 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:40.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.126261 25057 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:40.128280 24689 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:40.130455 24689 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:40.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.135855 24976 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" is_pre_election: true
00:20:40.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.136372 24976 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cfbe5cfb9f79473db877ca144fd09767 in term 7.
00:20:40.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.137395 24687 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc
00:20:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.137857 25057 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 8 FOLLOWER]: Leader pre-election won for term 9
00:20:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.138021 25057 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 48b11ebe2f3b40e98ce801db2d63ffdc)
00:20:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.138137 25057 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 8 FOLLOWER]: Advancing to term 9
00:20:40.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.140609 25057 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:40.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.141449 25057 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 9 election: Requested vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:40.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.142145 24976 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:40.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.142421 24976 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 7 FOLLOWER]: Advancing to term 9
00:20:40.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:40.143224 24689 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111)
00:20:40.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.146418 24976 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cfbe5cfb9f79473db877ca144fd09767 in term 9.
00:20:40.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.147010 24687 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc
00:20:40.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.147496 25057 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Leader election won for term 9
00:20:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.148046 25057 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 LEADER]: Becoming Leader. State: Replica: cfbe5cfb9f79473db877ca144fd09767, State: Running, Role: LEADER
00:20:40.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.148707 25057 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:40.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.153198 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 reported cstate change: term changed from 8 to 9, leader changed from 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65) to cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66). New cstate: current_term: 9 leader_uuid: "cfbe5cfb9f79473db877ca144fd09767" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: UNKNOWN } } }
00:20:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:40.686209 24689 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 -> Peer 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753): Couldn't send request to peer 48b11ebe2f3b40e98ce801db2d63ffdc. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.65:33753: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:40.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.731297 24976 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 9 FOLLOWER]: Refusing update from remote peer cfbe5cfb9f79473db877ca144fd09767: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
00:20:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.732786 25079 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
00:20:40.824 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:40.825 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:40.825 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2177), Traces: [0ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [2ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [2ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [17ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [19ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [19ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [37ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [38ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [39ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [56ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [57ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [59ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [59ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [76ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [77ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [79ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [79ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [452ms] refreshing cache from master, [452ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [455ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [456ms] sending RPC to server 48b11ebe2f3b40e98ce801db2d63ffdc, [458ms] received response from server 48b11ebe2f3b40e98ce801db2d63ffdc: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [458ms] delaying RPC due to: Network error: Failed to connect to peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753): Connection refused: /127.22.28.65:33753, [2173ms] refreshing cache from master, [2173ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [2176ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@135170592(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@122d22f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742) -> (continuation of Deferred@1909384575 after retry RPC@113347751) -> (continuation of Deferred@1455789927 after retry RPC@53601284) -> (continuation of Deferred@58763270 after retry RPC@261975478), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a24649a -> wakeup thread Time-limited test -> (continuation of Deferred@1743332808 after retry RPC@234917147) -> (continuation of Deferred@1113971452 after retry RPC@1208479742) -> (continuation of Deferred@1909384575 after retry RPC@113347751) -> (continuation of Deferred@1455789927 after retry RPC@53601284) -> (continuation of Deferred@58763270 after retry RPC@261975478))) after lookup
00:20:40.826 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:40.827 [DEBUG - kudu-nio-3] (Connection.java:757) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] Successfully connected to peer
00:20:40.844 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x44366d3c, L:/127.0.0.1:46166 - R:/127.22.28.66:40559] using TOKEN/null
00:20:40.871 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.65:33753
00:20:40.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:20:40.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:40.872680 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:33753
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44085
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:41.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.265229 25096 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:41.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.265604 25096 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:41.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.265684 25096 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.274467 25096 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.274767 25096 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.65
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.285873 25096 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/logs
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.65:33753
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.65
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44085
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.65
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:41.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:41.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:41.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.287849 25096 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:41.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.289950 25096 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.301724 25104 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.301801 25107 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.301695 25103 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:41.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.303107 25096 server_base.cc:1061] running on GCE node
00:20:41.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.303843 25096 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:41.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.305044 25096 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.306300 25096 hybrid_clock.cc:648] HybridClock initialized: now 1786494041306225 us; error 62 us; skew 500 ppm
00:20:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.306694 25096 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:41.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.309440 25096 webserver.cc:533] Webserver started at http://127.22.28.65:44085/ using document root <none> and password file <none>
00:20:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.310436 25096 fs_manager.cc:362] Metadata directory not provided
00:20:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.310597 25096 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.317476 25096 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.002s	sys 0.001s
00:20:41.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.321378 25115 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:41.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.323042 25096 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.323282 25096 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:08 on dist-test-slave-8w9v"
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.324374 25096 fs_report.cc:389] FS layout report
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/wal
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/data
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.348575 25096 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.349792 25096 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.350252 25096 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.351562 25096 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.355077 25122 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.362126 25096 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.362254 25096 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.008s	user 0.001s	sys 0.000s
00:20:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.362380 25096 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.365819 25096 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.365940 25096 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.002s	sys 0.000s
00:20:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.366626 25122 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap starting.
00:20:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.396618 25122 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Log is configured to *not* fsync() on all Append() calls
00:20:41.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.422857 25096 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.65:33753
00:20:41.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.422923 25256 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.65:33753 every 8 connection(s)
00:20:41.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.424571 25096 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-0/data/info.pb
00:20:41.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.428247 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 25096
00:20:41.429 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:41.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.440693 25122 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:41.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.441689 25122 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Bootstrap complete.
00:20:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.443486 25122 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent bootstrapping tablet: real 0.077s	user 0.055s	sys 0.020s
00:20:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.451051 25122 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.452175 25122 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Initialized, Role: FOLLOWER
00:20:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.453356 25122 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:41.455 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:41.456 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.66:40559
00:20:41.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:20:41.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.457191 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 24666
00:20:41.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.462568 25122 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc: Time spent starting tablet: real 0.019s	user 0.010s	sys 0.008s
00:20:41.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.491696 24817 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.494556 25257 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.495119 25257 heartbeater.cc:461] Registering TS with master...
00:20:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.496510 25257 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:41.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.500818 22863 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:41.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.501392 25258 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:41.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.501605 25258 heartbeater.cc:461] Registering TS with master...
00:20:41.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.502720 25258 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.509132 22691 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.510816 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.65:53123
00:20:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.512373 25259 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.512560 25259 heartbeater.cc:461] Registering TS with master...
00:20:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.513269 25259 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.514859 22785 ts_manager.cc:194] Re-registered known tserver with Master: 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.514900 25258 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:41.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:41.619144 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 24666
00:20:41.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:41.647 [DEBUG - kudu-nio-3] (Connection.java:239) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] handling channelInactive
00:20:41.647 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state READY due to: connection closed
00:20:41.648 [DEBUG - Time-limited test] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:41.648443 25276 negotiation.cc:336] Failed RPC negotiation. Trace:
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.621035 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.22.28.66:49093 (local address 127.22.28.65:33753)
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.624727 (+  3692us) server_negotiation.cc:207] Beginning negotiation
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.624739 (+    12us) server_negotiation.cc:400] Waiting for connection header
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.624783 (+    44us) server_negotiation.cc:408] Connection header received
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.624910 (+   127us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.624922 (+    12us) server_negotiation.cc:462] Received NEGOTIATE request from client
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.625238 (+   316us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
00:20:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.625482 (+   244us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
00:20:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 00:20:41.647198 (+ 21716us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.22.28.66:49093: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:20:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3468,"thread_start_us":3295,"threads_started":1}
00:20:41.649 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.649 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.649 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.664 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:41.664 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:41.664 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [2ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@609883841(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@32054e57 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3797cc78 -> wakeup thread Time-limited test)) after lookup
00:20:41.665 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:41.666 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.666 [DEBUG - kudu-nio-5] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.666 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.684 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:41.684 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:41.684 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [2ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [18ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [19ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [19ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [36ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@609883841(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@32054e57 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3797cc78 -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147))) after lookup
00:20:41.685 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:41.686 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.686 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.686 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.723 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:41.724 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:41.724 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [2ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [18ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [19ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [19ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [36ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [39ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [39ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [76ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@609883841(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@32054e57 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3797cc78 -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720))) after lookup
00:20:41.724 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:41.725 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.725 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.726 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.744 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:41.745 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:41.745 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=97), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [2ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [18ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [19ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [19ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [36ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [39ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [39ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [76ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [77ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [78ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [79ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [97ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@609883841(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@32054e57 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720) -> (continuation of Deferred@58290408 after retry RPC@51513388), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3797cc78 -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720) -> (continuation of Deferred@58290408 after retry RPC@51513388))) after lookup
00:20:41.746 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:41.747 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.747 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:41.748 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:42.084 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:42.085 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)[L]]}]
00:20:42.085 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=437), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [2ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [18ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [19ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [19ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [36ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [39ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [39ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [76ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [77ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [78ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [79ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [97ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [99ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [101ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [101ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [434ms] refreshing cache from master, [434ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [437ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@609883841(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@32054e57 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720) -> (continuation of Deferred@58290408 after retry RPC@51513388) -> (continuation of Deferred@1489001536 after retry RPC@1258318876), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3797cc78 -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720) -> (continuation of Deferred@58290408 after retry RPC@51513388) -> (continuation of Deferred@1489001536 after retry RPC@1258318876))) after lookup
00:20:42.085 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] connecting to peer
00:20:42.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:42.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] cleaning up while in state CONNECTING due to: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:42.087 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559
00:20:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.620390 25277 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:20:42.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.620705 25277 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.622588 25277 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:42.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:42.634840 25116 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:42.637180 25116 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:42.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.641722 24976 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" is_pre_election: true
00:20:42.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.642829 25117 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc; no voters: c2f0ab76e269428f9c0201c5fbe88e37, cfbe5cfb9f79473db877ca144fd09767
00:20:42.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.643344 25277 raft_consensus.cc:2749] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
00:20:42.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.795583 25283 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader cfbe5cfb9f79473db877ca144fd09767)
00:20:42.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.795862 25283 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.796854 25283 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:42.799585 24895 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.805665 25200 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" is_pre_election: true
00:20:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.806150 25200 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 8.
00:20:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.807214 24898 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37; no voters: cfbe5cfb9f79473db877ca144fd09767
00:20:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.807602 25283 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 9 FOLLOWER]: Leader pre-election won for term 10
00:20:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.807749 25283 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 9 FOLLOWER]: Starting leader election (detected failure of leader cfbe5cfb9f79473db877ca144fd09767)
00:20:42.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.807911 25283 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 9 FOLLOWER]: Advancing to term 10
00:20:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.810156 25283 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.810947 25283 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 10 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.811687 25200 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc"
00:20:42.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.811961 25200 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 8 FOLLOWER]: Advancing to term 10
00:20:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:42.812842 24895 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111)
00:20:42.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.816536 25200 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c2f0ab76e269428f9c0201c5fbe88e37 in term 10.
00:20:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.817126 24898 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37; no voters: cfbe5cfb9f79473db877ca144fd09767
00:20:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.817497 25283 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 10 FOLLOWER]: Leader election won for term 10
00:20:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.818004 25283 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 10 LEADER]: Becoming Leader. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Running, Role: LEADER
00:20:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.818596 25283 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.823073 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 reported cstate change: term changed from 9 to 10, leader changed from cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66) to c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67). New cstate: current_term: 10 leader_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: UNKNOWN } } }
00:20:42.864 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:42.864 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:42.865 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1217), Traces: [0ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [2ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [2ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [18ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [19ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [19ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [36ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [38ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [39ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [39ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [76ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [77ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [78ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [79ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [97ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [99ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [101ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [101ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [434ms] refreshing cache from master, [434ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [437ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [438ms] sending RPC to server cfbe5cfb9f79473db877ca144fd09767, [440ms] received response from server cfbe5cfb9f79473db877ca144fd09767: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [440ms] delaying RPC due to: Network error: Failed to connect to peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559): Connection refused: /127.22.28.66:40559, [1213ms] refreshing cache from master, [1214ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [1217ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@609883841(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@32054e57 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720) -> (continuation of Deferred@58290408 after retry RPC@51513388) -> (continuation of Deferred@1489001536 after retry RPC@1258318876) -> (continuation of Deferred@268624904 after retry RPC@1874641296), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3797cc78 -> wakeup thread Time-limited test -> (continuation of Deferred@1786924911 after retry RPC@1669315147) -> (continuation of Deferred@684038409 after retry RPC@922145720) -> (continuation of Deferred@58290408 after retry RPC@51513388) -> (continuation of Deferred@1489001536 after retry RPC@1258318876) -> (continuation of Deferred@268624904 after retry RPC@1874641296))) after lookup
00:20:42.865 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:42.866 [DEBUG - kudu-nio-2] (Connection.java:757) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] Successfully connected to peer
00:20:42.883 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xa731b62b, L:/127.0.0.1:33670 - R:/127.22.28.67:41901] using TOKEN/null
00:20:42.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.889981 25200 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 10 FOLLOWER]: Refusing update from remote peer c2f0ab76e269428f9c0201c5fbe88e37: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
00:20:42.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:42.890265 24895 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 -> Peer cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559): Couldn't send request to peer cfbe5cfb9f79473db877ca144fd09767. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.66:40559: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:42.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.890978 25288 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
00:20:42.923 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.66:40559
00:20:42.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:42.923942 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:40559
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41155
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:43.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.307209 25313 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:43.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.307704 25313 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:43.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.307787 25313 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.316725 25313 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:43.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.316911 25313 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.66
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.327895 25313 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/logs
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.66:40559
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.66
00:20:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41155
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.66
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.330278 25313 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.333103 25313 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:43.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.344859 25319 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:43.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.344859 25318 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:43.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:43.346251 25321 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:43.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.346514 25313 server_base.cc:1061] running on GCE node
00:20:43.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.347172 25313 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:43.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.348239 25313 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:43.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.349527 25313 hybrid_clock.cc:648] HybridClock initialized: now 1786494043349454 us; error 56 us; skew 500 ppm
00:20:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.349947 25313 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:43.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.352579 25313 webserver.cc:533] Webserver started at http://127.22.28.66:41155/ using document root <none> and password file <none>
00:20:43.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.353518 25313 fs_manager.cc:362] Metadata directory not provided
00:20:43.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.353673 25313 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:43.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.359812 25313 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.006s	sys 0.000s
00:20:43.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.364403 25327 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:43.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.366079 25313 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.366318 25313 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.367234 25313 fs_report.cc:389] FS layout report
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/wal
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/data
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:43.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:43.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:43.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:43.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.391919 25313 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:43.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.393071 25313 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:43.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.393492 25313 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:43.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.394590 25313 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:43.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.397151 25339 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:43.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.403985 25313 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:43.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.404174 25313 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:20:43.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.404323 25313 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.407747 25313 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:43.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.407908 25313 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.000s
00:20:43.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.408565 25339 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap starting.
00:20:43.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.444051 25339 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Log is configured to *not* fsync() on all Append() calls
00:20:43.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.487133 25313 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.66:40559
00:20:43.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.487172 25469 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.66:40559 every 8 connection(s)
00:20:43.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.488967 25313 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-1/data/info.pb
00:20:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.489810 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 25313
00:20:43.491 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:43.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.495361 25339 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:43.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.496250 25339 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Bootstrap complete.
00:20:43.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.497870 25339 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent bootstrapping tablet: real 0.090s	user 0.063s	sys 0.019s
00:20:43.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.506493 25339 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.507669 25339 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: cfbe5cfb9f79473db877ca144fd09767, State: Initialized, Role: FOLLOWER
00:20:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.509433 25339 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:43.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.519264 25339 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767: Time spent starting tablet: real 0.021s	user 0.014s	sys 0.005s
00:20:43.526 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:43.531 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.28.67:41901
00:20:43.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:43.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.531849 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 24877
00:20:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.583315 25470 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.583729 25470 heartbeater.cc:461] Registering TS with master...
00:20:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.583357 25478 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.584905 25478 heartbeater.cc:461] Registering TS with master...
00:20:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.585152 25470 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.585554 25478 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.588775 22784 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.590155 22863 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:43.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.593731 25029 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.596344 25471 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.596565 25471 heartbeater.cc:461] Registering TS with master...
00:20:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.597131 25471 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.598867 22691 ts_manager.cc:194] Re-registered known tserver with Master: cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559)
00:20:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.601233 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.66:44375
00:20:43.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:43.705263 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 24877
00:20:43.732 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:43.733 [DEBUG - kudu-nio-2] (Connection.java:239) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] handling channelInactive
00:20:43.733 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state READY due to: connection closed
00:20:43.733 [DEBUG - Time-limited test] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:43.734 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.734 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.734 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.744 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:43.744 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:43.745 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [1ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [1ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [10ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1743509678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6e523582 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20bb842d -> wakeup thread Time-limited test)) after lookup
00:20:43.745 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:43.746 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.746 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.746 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.763 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:43.764 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:43.764 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [1ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [1ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [10ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [12ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [13ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [13ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1743509678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6e523582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20bb842d -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917))) after lookup
00:20:43.765 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:43.766 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.766 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.766 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.804 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:43.804 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:43.805 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [1ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [1ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [10ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [12ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [13ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [13ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [31ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [33ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [33ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [71ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1743509678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6e523582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20bb842d -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190))) after lookup
00:20:43.805 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:43.807 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.807 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.807 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.864 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:43.865 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:43.865 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=132), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [1ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [1ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [10ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [12ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [13ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [13ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [31ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [33ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [33ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [71ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [72ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [74ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [74ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [131ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1743509678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6e523582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190) -> (continuation of Deferred@831410239 after retry RPC@329080465), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20bb842d -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190) -> (continuation of Deferred@831410239 after retry RPC@329080465))) after lookup
00:20:43.865 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:43.867 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.867 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:43.867 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:44.383 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:44.384 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753),c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)[L],cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:44.384 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=651), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [1ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [1ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [10ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [12ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [13ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [13ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [31ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [33ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [33ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [71ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [72ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [74ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [74ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [131ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [132ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [134ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [134ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [647ms] refreshing cache from master, [647ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [650ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1743509678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6e523582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190) -> (continuation of Deferred@831410239 after retry RPC@329080465) -> (continuation of Deferred@1693308201 after retry RPC@494757619), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20bb842d -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190) -> (continuation of Deferred@831410239 after retry RPC@329080465) -> (continuation of Deferred@1693308201 after retry RPC@494757619))) after lookup
00:20:44.385 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] connecting to peer
00:20:44.386 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:44.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] cleaning up while in state CONNECTING due to: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:44.386 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901) for tablet dffc135da8c24a56a4d38372feab7346: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901
00:20:44.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.605664 25471 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.878600 25490 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:20:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.878911 25490 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.880834 25490 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901), 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65:33753)
00:20:44.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:44.882547 25330 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:44.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:44.885149 25330 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:44.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.888844 25200 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "cfbe5cfb9f79473db877ca144fd09767" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" is_pre_election: true
00:20:44.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.889711 25332 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cfbe5cfb9f79473db877ca144fd09767; no voters: 48b11ebe2f3b40e98ce801db2d63ffdc, c2f0ab76e269428f9c0201c5fbe88e37
00:20:44.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.890139 25490 raft_consensus.cc:2749] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
00:20:44.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.951850 25496 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 10 FOLLOWER]: Starting pre-election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:44.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.952072 25496 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:44.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.953055 25496 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:44.955977 25117 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.962059 25417 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767" is_pre_election: true
00:20:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.962484 25417 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 9.
00:20:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.963483 25116 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.963831 25496 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 10 FOLLOWER]: Leader pre-election won for term 11
00:20:44.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.963965 25496 raft_consensus.cc:493] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 10 FOLLOWER]: Starting leader election (detected failure of leader c2f0ab76e269428f9c0201c5fbe88e37)
00:20:44.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.964056 25496 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 10 FOLLOWER]: Advancing to term 11
00:20:44.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.966276 25496 raft_consensus.cc:515] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:44.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.967082 25496 leader_election.cc:290] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 11 election: Requested vote from peers cfbe5cfb9f79473db877ca144fd09767 (127.22.28.66:40559), c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:44.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.967653 25417 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "dffc135da8c24a56a4d38372feab7346" candidate_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "cfbe5cfb9f79473db877ca144fd09767"
00:20:44.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.967934 25417 raft_consensus.cc:3060] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 9 FOLLOWER]: Advancing to term 11
00:20:44.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:44.969075 25117 leader_election.cc:336] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111)
00:20:44.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.972455 25417 raft_consensus.cc:2468] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 48b11ebe2f3b40e98ce801db2d63ffdc in term 11.
00:20:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.973117 25116 leader_election.cc:304] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 48b11ebe2f3b40e98ce801db2d63ffdc, cfbe5cfb9f79473db877ca144fd09767; no voters: c2f0ab76e269428f9c0201c5fbe88e37
00:20:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.973567 25496 raft_consensus.cc:2804] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 11 FOLLOWER]: Leader election won for term 11
00:20:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.974107 25496 raft_consensus.cc:697] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [term 11 LEADER]: Becoming Leader. State: Replica: 48b11ebe2f3b40e98ce801db2d63ffdc, State: Running, Role: LEADER
00:20:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.974696 25496 consensus_queue.cc:237] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:44.978958 22691 catalog_manager.cc:5719] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc reported cstate change: term changed from 10 to 11, leader changed from c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67) to 48b11ebe2f3b40e98ce801db2d63ffdc (127.22.28.65). New cstate: current_term: 11 leader_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } health_report { overall_health: HEALTHY } } }
00:20:45.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:45.421514 25117 consensus_peers.cc:597] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc -> Peer c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901): Couldn't send request to peer c2f0ab76e269428f9c0201c5fbe88e37. Status: Network error: Client connection negotiation failed: client connection to 127.22.28.67:41901: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:20:45.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:45.574319 25417 raft_consensus.cc:1275] T dffc135da8c24a56a4d38372feab7346 P cfbe5cfb9f79473db877ca144fd09767 [term 11 FOLLOWER]: Refusing update from remote peer 48b11ebe2f3b40e98ce801db2d63ffdc: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
00:20:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:45.575388 25496 consensus_queue.cc:1048] T dffc135da8c24a56a4d38372feab7346 P 48b11ebe2f3b40e98ce801db2d63ffdc [LEADER]: Connected to new peer: Peer: permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
00:20:45.744 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dffc135da8c24a56a4d38372feab7346 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205' with partition [<start>, <end>)
00:20:45.744 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=dffc135da8c24a56a4d38372feab7346@[48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)[L],c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901),cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)]}]
00:20:45.744 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=dffc135da8c24a56a4d38372feab7346, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2011), Traces: [0ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [1ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [1ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [10ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [12ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [13ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [13ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [30ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [31ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [33ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [33ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [71ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [72ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [74ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [74ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [131ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [132ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [134ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [134ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [647ms] refreshing cache from master, [647ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [650ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, [651ms] sending RPC to server c2f0ab76e269428f9c0201c5fbe88e37, [653ms] received response from server c2f0ab76e269428f9c0201c5fbe88e37: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [653ms] delaying RPC due to: Network error: Failed to connect to peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901): Connection refused: /127.22.28.67:41901, [2007ms] refreshing cache from master, [2007ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.28.126:40901, [2010ms] Sub RPC GetTableLocations: received response from server master-127.22.28.126:40901: OK, deferred=Deferred@1743509678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f78e836b560@6e523582 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190) -> (continuation of Deferred@831410239 after retry RPC@329080465) -> (continuation of Deferred@1693308201 after retry RPC@494757619) -> (continuation of Deferred@2012643101 after retry RPC@30070254), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@20bb842d -> wakeup thread Time-limited test -> (continuation of Deferred@1952521896 after retry RPC@134706917) -> (continuation of Deferred@1912600866 after retry RPC@1115425190) -> (continuation of Deferred@831410239 after retry RPC@329080465) -> (continuation of Deferred@1693308201 after retry RPC@494757619) -> (continuation of Deferred@2012643101 after retry RPC@30070254))) after lookup
00:20:45.745 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] connecting to peer
00:20:45.746 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] Successfully connected to peer
00:20:45.765 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x0f032f5a, L:/127.0.0.1:46032 - R:/127.22.28.65:33753] using TOKEN/null
00:20:45.791 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.28.67:41901
00:20:45.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:45.792446 22641 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_i0RbK/build/asan/bin/kudu
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:20:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:20:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.181661 25534 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:20:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.182034 25534 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:20:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.182113 25534 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:20:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.191075 25534 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:20:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.191272 25534 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.28.67
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.201982 25534 tablet_server_runner.cc:78] Tablet server non-default flags:
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/logs
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.28.84:35779
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.28.67:41901
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.28.67
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38727
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.28.126:40901,127.22.28.125:43953,127.22.28.124:44295
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.28.67
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:20:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:20:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:20:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:09 UTC on bdcb31816ec0
00:20:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17638
00:20:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
00:20:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.203876 25534 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:20:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.206041 25534 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:20:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.217769 25541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.217726 25545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:20:46.217809 25542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:20:46.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.219224 25534 server_base.cc:1061] running on GCE node
00:20:46.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.220009 25534 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:20:46.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.221261 25534 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:20:46.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.222527 25534 hybrid_clock.cc:648] HybridClock initialized: now 1786494046222473 us; error 40 us; skew 500 ppm
00:20:46.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.222932 25534 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:20:46.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.225586 25534 webserver.cc:533] Webserver started at http://127.22.28.67:38727/ using document root <none> and password file <none>
00:20:46.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.226577 25534 fs_manager.cc:362] Metadata directory not provided
00:20:46.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.226733 25534 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:20:46.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.233214 25534 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
00:20:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.236974 25551 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:20:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.238561 25534 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.238793 25534 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data,/tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2f0ab76e269428f9c0201c5fbe88e37"
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:20:09 on dist-test-slave-8w9v"
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.239938 25534 fs_report.cc:389] FS layout report
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/wal
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/data
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:20:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:20:46.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.256305 25534 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:20:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.257429 25534 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:20:46.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.257833 25534 kserver.cc:163] Server-wide thread pool size limit: 3276
00:20:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.259050 25534 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:20:46.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.261641 25558 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:20:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.268025 25534 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:20:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.268229 25534 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:20:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.268426 25534 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:20:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.271771 25534 ts_tablet_manager.cc:616] Registered 1 tablets
00:20:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.271927 25534 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.000s	sys 0.000s
00:20:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.272535 25558 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap starting.
00:20:46.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.311182 25558 log.cc:826] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Log is configured to *not* fsync() on all Append() calls
00:20:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.328573 25534 rpc_server.cc:307] RPC server started. Bound to: 127.22.28.67:41901
00:20:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.328573 25683 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.28.67:41901 every 8 connection(s)
00:20:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.330199 25534 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_i0RbK/test-tmp/mini-kudu-cluster214715210376526910/ts-2/data/info.pb
00:20:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.338764 22641 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_i0RbK/build/asan/bin/kudu as pid 25534
00:20:46.339 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:20:46.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.360134 25690 heartbeater.cc:344] Connected to a master server at 127.22.28.125:43953
00:20:46.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.360562 25690 heartbeater.cc:461] Registering TS with master...
00:20:46.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.362013 25690 heartbeater.cc:507] Master 127.22.28.125:43953 requested a full tablet report, sending...
00:20:46.364 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.365821 22784 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:46.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.367615 25558 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:20:46.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.368449 25558 tablet_bootstrap.cc:492] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Bootstrap complete.
00:20:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.370398 25558 ts_tablet_manager.cc:1403] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent bootstrapping tablet: real 0.098s	user 0.079s	sys 0.013s
00:20:46.373 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786494010205, tablet=dffc135da8c24a56a4d38372feab7346, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:20:46.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.377858 25558 raft_consensus.cc:359] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:46.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.379041 25558 raft_consensus.cc:740] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: c2f0ab76e269428f9c0201c5fbe88e37, State: Initialized, Role: FOLLOWER
00:20:46.379 [DEBUG - main] (Connection.java:575) [peer master-127.22.28.126:40901(127.22.28.126:40901)] disconnecting while in state READY
00:20:46.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.379998 25558 consensus_queue.cc:260] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cfbe5cfb9f79473db877ca144fd09767" member_type: VOTER last_known_addr { host: "127.22.28.66" port: 40559 } } peers { permanent_uuid: "c2f0ab76e269428f9c0201c5fbe88e37" member_type: VOTER last_known_addr { host: "127.22.28.67" port: 41901 } } peers { permanent_uuid: "48b11ebe2f3b40e98ce801db2d63ffdc" member_type: VOTER last_known_addr { host: "127.22.28.65" port: 33753 } }
00:20:46.380 [DEBUG - main] (Connection.java:575) [peer master-127.22.28.124:44295(127.22.28.124:44295)] disconnecting while in state READY
00:20:46.381 [DEBUG - main] (Connection.java:575) [peer c2f0ab76e269428f9c0201c5fbe88e37(127.22.28.67:41901)] disconnecting while in state TERMINATED
00:20:46.381 [DEBUG - main] (Connection.java:575) [peer cfbe5cfb9f79473db877ca144fd09767(127.22.28.66:40559)] disconnecting while in state TERMINATED
00:20:46.381 [DEBUG - main] (Connection.java:575) [peer master-127.22.28.125:43953(127.22.28.125:43953)] disconnecting while in state READY
00:20:46.381 [DEBUG - main] (Connection.java:575) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] disconnecting while in state READY
00:20:46.383 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.22.28.124:44295(127.22.28.124:44295)] handling channelInactive
00:20:46.383 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.28.125:43953(127.22.28.125:43953)] handling channelInactive
00:20:46.383 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.22.28.124:44295(127.22.28.124:44295)] cleaning up while in state READY due to: connection closed
00:20:46.383 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.28.125:43953(127.22.28.125:43953)] cleaning up while in state READY due to: connection closed
00:20:46.385 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:20:46.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.383652 25687 heartbeater.cc:344] Connected to a master server at 127.22.28.126:40901
00:20:46.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.383888 25687 heartbeater.cc:461] Registering TS with master...
00:20:46.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.384735 25687 heartbeater.cc:507] Master 127.22.28.126:40901 requested a full tablet report, sending...
00:20:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.386981 22691 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:46.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.388731 22691 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.28.67:54389
00:20:46.391 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.28.126:40901(127.22.28.126:40901)] handling channelInactive
00:20:46.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.28.126:40901(127.22.28.126:40901)] cleaning up while in state READY due to: connection closed
00:20:46.392 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] handling channelInactive
00:20:46.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 48b11ebe2f3b40e98ce801db2d63ffdc(127.22.28.65:33753)] cleaning up while in state READY due to: connection closed
00:20:46.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.393687 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 25096
00:20:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.396346 25686 heartbeater.cc:344] Connected to a master server at 127.22.28.124:44295
00:20:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.396554 25686 heartbeater.cc:461] Registering TS with master...
00:20:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.397157 25686 heartbeater.cc:507] Master 127.22.28.124:44295 requested a full tablet report, sending...
00:20:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.399209 22863 ts_manager.cc:194] Re-registered known tserver with Master: c2f0ab76e269428f9c0201c5fbe88e37 (127.22.28.67:41901)
00:20:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.404510 25558 ts_tablet_manager.cc:1434] T dffc135da8c24a56a4d38372feab7346 P c2f0ab76e269428f9c0201c5fbe88e37: Time spent starting tablet: real 0.034s	user 0.012s	sys 0.013s
00:20:46.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.405117 25687 heartbeater.cc:499] Master 127.22.28.126:40901 was elected leader, sending a full tablet report...
00:20:46.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.415324 25252 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:46.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.518545 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 25096
00:20:46.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.546298 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 25313
00:20:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.554653 25465 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:46.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.653391 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 25313
00:20:46.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.680824 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 25534
00:20:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.690057 25679 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.784502 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 25534
00:20:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.810912 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 22655
00:20:46.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.812426 22732 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:46.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.944490 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 22655
00:20:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.979413 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 22742
00:20:46.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:46.980803 22817 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:47.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:47.108572 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 22742
00:20:47.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:47.141575 22641 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-task_i0RbK/build/asan/bin/kudu pid 22826
00:20:47.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:47.142982 22904 generic_service.cc:196] Checking for leaks (request via RPC)
00:20:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:20:47.270879 22641 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_i0RbK/build/asan/bin/kudu with pid 22826
00:20:47.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:20:47Z chronyd exiting

Time: 42.589

OK (1 test)

