JUnit version 4.13.2
.23:59:01.741 [INFO - main] (RandomUtils.java:49) Using random seed: 1783468741736
23:59:01.785 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
23:59:01.835 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:59:01.847 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task84Dvx3/test-tmp
23:59:01.850 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
23:59:01.853 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task84Dvx3/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
23:59:02.117 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

23:59:02.266 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:59:02.267 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

23:59:02.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.270227 20244 mini_oidc.cc:152] Starting JWKS server
23:59:02.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.272847 20244 webserver.cc:460] Webserver started at http://127.0.0.1:45691/ using document root <none> and password file <none>
23:59:02.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.273067 20244 mini_oidc.cc:189] Starting OIDC Discovery server
23:59:02.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.273720 20244 webserver.cc:460] Webserver started at http://127.0.0.1:37137/ using document root <none> and password file <none>
23:59:02.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:59:02Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:59:02.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:59:02Z Disabled control of system clock
23:59:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.302140 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/wal
23:59:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data
23:59:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/logs
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data/info.pb
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:59:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.62:35395
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.62
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:59:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:59:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45691/jwks.json with env {}
23:59:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.618939 20255 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.619215 20255 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.619297 20255 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:02.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.626549 20255 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:59:02.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.626670 20255 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:02.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.626718 20255 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:59:02.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.626767 20255 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:59:02.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.626804 20255 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:59:02.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.626852 20255 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45691/jwks.json
23:59:02.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.641256 20255 master_runner.cc:386] Master server non-default flags:
23:59:02.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:02.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:02.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data
23:59:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/wal
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.62:35395
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:59:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45691/jwks.json
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data/info.pb
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.62
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/logs
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.642953 20255 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.645284 20255 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.658020 20263 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.657966 20260 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.657991 20261 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:02.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.658833 20255 server_base.cc:1034] running on GCE node
23:59:02.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.659621 20255 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.660902 20255 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.662178 20255 hybrid_clock.cc:648] HybridClock initialized: now 1783468742662098 us; error 49 us; skew 500 ppm
23:59:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.662578 20255 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.664839 20255 webserver.cc:460] Webserver started at http://127.19.197.62:35969/ using document root <none> and password file <none>
23:59:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.665556 20255 fs_manager.cc:359] Metadata directory not provided
23:59:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.665701 20255 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:02.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.666103 20255 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:59:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.668740 20255 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data/instance:
23:59:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "300311eef5af4f7fa51959f4df1a1b83"
23:59:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:02 on dist-test-slave-x4hg"
23:59:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.669629 20255 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/wal/instance:
23:59:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "300311eef5af4f7fa51959f4df1a1b83"
23:59:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:02 on dist-test-slave-x4hg"
23:59:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.674650 20255 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.002s
23:59:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.677989 20269 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.679369 20255 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
23:59:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.679553 20255 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/wal
23:59:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "300311eef5af4f7fa51959f4df1a1b83"
23:59:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:02 on dist-test-slave-x4hg"
23:59:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.679821 20255 fs_report.cc:383] FS layout report
23:59:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/wal
23:59:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/wal
23:59:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data/data
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:02.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.702232 20255 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:02.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.703188 20255 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:02.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.703567 20255 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.721328 20255 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.62:35395
23:59:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.721366 20320 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.62:35395 every 8 connection(s)
23:59:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.722886 20255 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/data/info.pb
23:59:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.726575 20321 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:59:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.727808 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 20255
23:59:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.728080 20244 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-0/wal/instance
23:59:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.730821 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/wal
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/logs
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data/info.pb
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.61:35737
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.61
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:02.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:59:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45691/jwks.json with env {}
23:59:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.735690 20321 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } has no permanent_uuid. Determining permanent_uuid...
23:59:02.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.754452 20321 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } has no permanent_uuid. Determining permanent_uuid...
23:59:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.755620 20272 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.61:35737: connect: Connection refused (error 111)
23:59:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.757061 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.61:35737: Network error: Client connection negotiation failed: client connection to 127.19.197.61:35737: connect: Connection refused (error 111)
23:59:02.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.806458 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } attempt: 1
23:59:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.808456 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.61:35737: Network error: Client connection negotiation failed: client connection to 127.19.197.61:35737: connect: Connection refused (error 111)
23:59:02.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.876705 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } attempt: 2
23:59:02.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.878816 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.61:35737: Network error: Client connection negotiation failed: client connection to 127.19.197.61:35737: connect: Connection refused (error 111)
23:59:02.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:02.970096 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } attempt: 3
23:59:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:02.972139 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.61:35737: Network error: Client connection negotiation failed: client connection to 127.19.197.61:35737: connect: Connection refused (error 111)
23:59:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.049904 20323 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.050308 20323 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.050447 20323 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.058782 20323 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:59:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.058928 20323 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.058996 20323 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:59:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.059069 20323 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:59:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.059126 20323 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:59:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.059206 20323 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45691/jwks.json
23:59:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.069830 20323 master_runner.cc:386] Master server non-default flags:
23:59:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data
23:59:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/wal
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.61:35737
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45691/jwks.json
23:59:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data/info.pb
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.61
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/logs
23:59:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:59:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:03.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.071659 20323 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:03.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.074225 20323 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.087182 20331 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.087131 20334 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.087147 20332 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:03.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.087886 20323 server_base.cc:1034] running on GCE node
23:59:03.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.088696 20323 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:03.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.090040 20323 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.091336 20323 hybrid_clock.cc:648] HybridClock initialized: now 1783468743091265 us; error 36 us; skew 500 ppm
23:59:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.091764 20323 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:03.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.094363 20323 webserver.cc:460] Webserver started at http://127.19.197.61:46569/ using document root <none> and password file <none>
23:59:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.095124 20323 fs_manager.cc:359] Metadata directory not provided
23:59:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.095285 20323 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.095700 20323 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:59:03.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.098691 20323 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data/instance:
23:59:03.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d231205e9a94388916806b12662111a"
23:59:03.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:03 on dist-test-slave-x4hg"
23:59:03.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.099668 20323 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/wal/instance:
23:59:03.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d231205e9a94388916806b12662111a"
23:59:03.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:03 on dist-test-slave-x4hg"
23:59:03.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.105269 20323 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.005s	sys 0.001s
23:59:03.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.109112 20340 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:03.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.110491 20323 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.110757 20323 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/wal
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d231205e9a94388916806b12662111a"
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:03 on dist-test-slave-x4hg"
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.111013 20323 fs_report.cc:383] FS layout report
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/wal
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/wal
23:59:03.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data/data
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.115362 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } attempt: 4
23:59:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.117424 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.61:35737: Network error: Client connection negotiation failed: client connection to 127.19.197.61:35737: connect: Connection refused (error 111)
23:59:03.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.131356 20323 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.132141 20323 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.132427 20323 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.151006 20323 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.61:35737
23:59:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.151029 20391 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.61:35737 every 8 connection(s)
23:59:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.152325 20323 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/data/info.pb
23:59:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.155875 20392 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:59:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.157231 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 20323
23:59:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.157428 20244 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-1/wal/instance
23:59:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.158460 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/wal
23:59:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/logs
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data/info.pb
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.60:43759
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.60
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:59:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:59:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45691/jwks.json with env {}
23:59:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.162283 20392 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } has no permanent_uuid. Determining permanent_uuid...
23:59:03.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.177693 20392 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } has no permanent_uuid. Determining permanent_uuid...
23:59:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.187945 20392 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } has no permanent_uuid. Determining permanent_uuid...
23:59:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.188968 20344 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.190412 20392 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.239696 20392 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 1
23:59:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.241665 20392 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.309880 20392 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 2
23:59:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.311939 20392 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.403168 20392 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 3
23:59:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.405349 20392 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.416674 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } attempt: 5
23:59:03.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.423768 20321 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } has no permanent_uuid. Determining permanent_uuid...
23:59:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.425475 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.476734 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 1
23:59:03.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.478677 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.488463 20394 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.488735 20394 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.488818 20394 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.495935 20394 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:59:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.496035 20394 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.496078 20394 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:59:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.496129 20394 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:59:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.496166 20394 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:59:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.496214 20394 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45691/jwks.json
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.505105 20394 master_runner.cc:386] Master server non-default flags:
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/wal
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.60:43759
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45691/jwks.json
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data/info.pb
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.60
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/logs
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.506708 20394 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.509042 20394 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.521152 20405 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.521250 20402 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.521682 20403 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.522042 20394 server_base.cc:1034] running on GCE node
23:59:03.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.522914 20394 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.524363 20394 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.525592 20394 hybrid_clock.cc:648] HybridClock initialized: now 1783468743525489 us; error 78 us; skew 500 ppm
23:59:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.525913 20394 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.528167 20394 webserver.cc:460] Webserver started at http://127.19.197.60:34589/ using document root <none> and password file <none>
23:59:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.528980 20394 fs_manager.cc:359] Metadata directory not provided
23:59:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.529101 20394 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.529372 20394 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:59:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.531957 20394 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data/instance:
23:59:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6640f710a7ce48eb96f01a08c4333169"
23:59:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:03 on dist-test-slave-x4hg"
23:59:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.532701 20394 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/wal/instance:
23:59:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6640f710a7ce48eb96f01a08c4333169"
23:59:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:03 on dist-test-slave-x4hg"
23:59:03.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.537742 20394 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.003s	sys 0.003s
23:59:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.540894 20411 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.542439 20394 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
23:59:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.542610 20394 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/wal
23:59:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6640f710a7ce48eb96f01a08c4333169"
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:03 on dist-test-slave-x4hg"
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.542795 20394 fs_report.cc:383] FS layout report
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/wal
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/wal
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data/data
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.546903 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 2
23:59:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.548566 20321 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.548560 20392 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 4
23:59:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.550192 20392 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.197.60:43759: Network error: Client connection negotiation failed: client connection to 127.19.197.60:43759: connect: Connection refused (error 111)
23:59:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.564664 20394 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:03.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.565393 20394 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:03.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.565675 20394 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:03.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.582333 20394 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.60:43759
23:59:03.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.582301 20462 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.60:43759 every 8 connection(s)
23:59:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.584060 20394 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/data/info.pb
23:59:03.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.587750 20463 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:59:03.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.593995 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 20394
23:59:03.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.594202 20244 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/master-2/wal/instance
23:59:03.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.593585 20463 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } has no permanent_uuid. Determining permanent_uuid...
23:59:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.608007 20463 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } has no permanent_uuid. Determining permanent_uuid...
23:59:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.617297 20463 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } has no permanent_uuid. Determining permanent_uuid...
23:59:03.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.628526 20463 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: Bootstrap starting.
23:59:03.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.631497 20463 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: Neither blocks nor log segments found. Creating new log.
23:59:03.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.632391 20463 log.cc:824] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: Log is configured to *not* fsync() on all Append() calls
23:59:03.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.634786 20463 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: No bootstrap required, opened a new log
23:59:03.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.642362 20463 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.642686 20463 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:59:03.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.642789 20463 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6640f710a7ce48eb96f01a08c4333169, State: Initialized, Role: FOLLOWER
23:59:03.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.643369 20463 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.644130 20472 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.644441 20472 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [sys.catalog]: This master's current role is: FOLLOWER
23:59:03.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.645145 20463 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [sys.catalog]: configured and running, proceeding with master startup.
23:59:03.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.654712 20483 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:59:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.654773 20321 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 3
23:59:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.655164 20483 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:59:03.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.651881 20394 master_runner.cc:186] Error getting master registration for 127.19.197.62:35395: OK, instance_id { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" instance_seqno: 1783468742716485 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:59:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.662277 20394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:59:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.670397 20321 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83: Bootstrap starting.
23:59:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.673990 20321 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83: Neither blocks nor log segments found. Creating new log.
23:59:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.675119 20321 log.cc:824] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83: Log is configured to *not* fsync() on all Append() calls
23:59:03.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.679209 20321 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83: No bootstrap required, opened a new log
23:59:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.690024 20321 raft_consensus.cc:348] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.690410 20321 raft_consensus.cc:374] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:59:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.690519 20321 raft_consensus.cc:729] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 300311eef5af4f7fa51959f4df1a1b83, State: Initialized, Role: FOLLOWER
23:59:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.691174 20321 consensus_queue.cc:260] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.691987 20486 sys_catalog.cc:455] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.692323 20486 sys_catalog.cc:458] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: This master's current role is: FOLLOWER
23:59:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.693388 20321 sys_catalog.cc:564] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: configured and running, proceeding with master startup.
23:59:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.702749 20497 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:59:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.704277 20497 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:59:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.706156 20275 tablet.cc:2369] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:59:03.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.701978 20255 master_runner.cc:186] Error getting master registration for 127.19.197.61:35737: OK, instance_id { permanent_uuid: "5d231205e9a94388916806b12662111a" instance_seqno: 1783468743145820 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:59:03.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.714003 20255 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:59:03.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.753412 20486 raft_consensus.cc:482] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:59:03.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.753720 20486 raft_consensus.cc:504] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.755406 20486 leader_election.cc:288] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5d231205e9a94388916806b12662111a (127.19.197.61:35737), 6640f710a7ce48eb96f01a08c4333169 (127.19.197.60:43759)
23:59:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.756522 20438 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "300311eef5af4f7fa51959f4df1a1b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6640f710a7ce48eb96f01a08c4333169" is_pre_election: true
23:59:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.757141 20438 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 300311eef5af4f7fa51959f4df1a1b83 in term 0.
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.757755 20273 leader_election.cc:302] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 300311eef5af4f7fa51959f4df1a1b83, 6640f710a7ce48eb96f01a08c4333169; no voters: 
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.758165 20486 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.758307 20486 raft_consensus.cc:482] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.758404 20486 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 0 FOLLOWER]: Advancing to term 1
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.761649 20486 raft_consensus.cc:504] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.762959 20438 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "300311eef5af4f7fa51959f4df1a1b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6640f710a7ce48eb96f01a08c4333169"
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.763154 20438 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [term 0 FOLLOWER]: Advancing to term 1
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.766345 20438 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 300311eef5af4f7fa51959f4df1a1b83 in term 1.
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.766752 20273 leader_election.cc:302] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 300311eef5af4f7fa51959f4df1a1b83, 6640f710a7ce48eb96f01a08c4333169; no voters: 
23:59:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.768908 20486 leader_election.cc:288] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [CANDIDATE]: Term 1 election: Requested vote from peers 5d231205e9a94388916806b12662111a (127.19.197.61:35737), 6640f710a7ce48eb96f01a08c4333169 (127.19.197.60:43759)
23:59:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.769109 20486 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 1 FOLLOWER]: Leader election won for term 1
23:59:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.769238 20486 raft_consensus.cc:686] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [term 1 LEADER]: Becoming Leader. State: Replica: 300311eef5af4f7fa51959f4df1a1b83, State: Running, Role: LEADER
23:59:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.769755 20486 consensus_queue.cc:237] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.786837 20499 sys_catalog.cc:455] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 300311eef5af4f7fa51959f4df1a1b83. Latest consensus state: current_term: 1 leader_uuid: "300311eef5af4f7fa51959f4df1a1b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.787595 20499 sys_catalog.cc:458] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: This master's current role is: LEADER
23:59:03.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.789068 20501 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
23:59:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.793253 20501 catalog_manager.cc:1484] Initializing Kudu cluster ID...
23:59:03.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.808892 20438 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [term 1 FOLLOWER]: Refusing update from remote peer 300311eef5af4f7fa51959f4df1a1b83: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:59:03.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.810211 20486 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:59:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:03.817545 20417 tablet.cc:2369] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:59:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.835479 20472 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 300311eef5af4f7fa51959f4df1a1b83. Latest consensus state: current_term: 1 leader_uuid: "300311eef5af4f7fa51959f4df1a1b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.835807 20472 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [sys.catalog]: This master's current role is: FOLLOWER
23:59:03.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.849400 20392 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } attempt: 5
23:59:03.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.851476 20499 sys_catalog.cc:455] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "300311eef5af4f7fa51959f4df1a1b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.852030 20499 sys_catalog.cc:458] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: This master's current role is: LEADER
23:59:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.858191 20501 catalog_manager.cc:1347] Generated new cluster ID: 06235fc4e144452eaeeea100ab14fdc2
23:59:03.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.858301 20501 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
23:59:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.865535 20472 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "300311eef5af4f7fa51959f4df1a1b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.865800 20472 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169 [sys.catalog]: This master's current role is: FOLLOWER
23:59:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.875695 20392 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a: Bootstrap starting.
23:59:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.882535 20392 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a: Neither blocks nor log segments found. Creating new log.
23:59:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.888641 20392 log.cc:824] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a: Log is configured to *not* fsync() on all Append() calls
23:59:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.901049 20392 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a: No bootstrap required, opened a new log
23:59:03.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.922238 20392 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.923384 20392 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:59:03.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.923815 20392 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d231205e9a94388916806b12662111a, State: Initialized, Role: FOLLOWER
23:59:03.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.925123 20392 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } }
23:59:03.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.939146 20501 catalog_manager.cc:1370] Generated new certificate authority record
23:59:03.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.940984 20366 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "300311eef5af4f7fa51959f4df1a1b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d231205e9a94388916806b12662111a"
23:59:03.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.941335 20365 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [term 0 FOLLOWER]: Advancing to term 1
23:59:03.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.942581 20392 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [sys.catalog]: configured and running, proceeding with master startup.
23:59:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.944845 20365 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [term 1 FOLLOWER]: Refusing update from remote peer 300311eef5af4f7fa51959f4df1a1b83: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:59:03.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.945592 20367 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "300311eef5af4f7fa51959f4df1a1b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d231205e9a94388916806b12662111a" is_pre_election: true
23:59:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.946947 20486 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:59:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.947700 20501 catalog_manager.cc:1504] Loading token signing keys...
23:59:03.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.950467 20508 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.951165 20508 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [sys.catalog]: This master's current role is: FOLLOWER
23:59:03.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.965170 20501 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83: Generated new TSK 0
23:59:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.966202 20501 catalog_manager.cc:1514] Initializing in-progress tserver states...
23:59:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.976925 20508 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 300311eef5af4f7fa51959f4df1a1b83. Latest consensus state: current_term: 1 leader_uuid: "300311eef5af4f7fa51959f4df1a1b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.977721 20508 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [sys.catalog]: This master's current role is: FOLLOWER
23:59:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.976548 20510 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "300311eef5af4f7fa51959f4df1a1b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.978247 20510 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a [sys.catalog]: This master's current role is: FOLLOWER
23:59:03.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.978799 20499 sys_catalog.cc:455] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "300311eef5af4f7fa51959f4df1a1b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "300311eef5af4f7fa51959f4df1a1b83" member_type: VOTER last_known_addr { host: "127.19.197.62" port: 35395 } } peers { permanent_uuid: "5d231205e9a94388916806b12662111a" member_type: VOTER last_known_addr { host: "127.19.197.61" port: 35737 } } peers { permanent_uuid: "6640f710a7ce48eb96f01a08c4333169" member_type: VOTER last_known_addr { host: "127.19.197.60" port: 43759 } } }
23:59:03.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.979045 20499 sys_catalog.cc:458] T 00000000000000000000000000000000 P 300311eef5af4f7fa51959f4df1a1b83 [sys.catalog]: This master's current role is: LEADER
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.988461 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:0
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:03.989152 20511 mvcc.cc:205] Tried to move back new op lower bound from 7305087975149211648 to 7305087974491750400. Current Snapshot: MvccSnapshot[applied={T|T < 7305087975149211648 or (T in {7305087975149211648})}]
23:59:04.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.030352 20525 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:59:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.374517 20519 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:04.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.374781 20519 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:04.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.374858 20519 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:04.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.381904 20519 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:04.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.382107 20519 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.1
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.391052 20519 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:0
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.392269 20519 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.394055 20519 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.407498 20531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:04.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.407583 20533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:04.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.407537 20530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:04.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.411383 20519 server_base.cc:1034] running on GCE node
23:59:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.412098 20519 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.413198 20519 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:04.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.414443 20519 hybrid_clock.cc:648] HybridClock initialized: now 1783468744414362 us; error 54 us; skew 500 ppm
23:59:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.415129 20519 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.417531 20519 webserver.cc:460] Webserver started at http://127.19.197.1:35163/ using document root <none> and password file <none>
23:59:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.418275 20519 fs_manager.cc:359] Metadata directory not provided
23:59:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.418421 20519 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.418741 20519 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:59:04.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.421298 20519 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/instance:
23:59:04.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:04.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.422179 20519 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal/instance:
23:59:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:04.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.427263 20519 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.004s	sys 0.000s
23:59:04.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.430510 20539 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:04.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.431787 20519 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.002s
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.431953 20519 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.432186 20519 fs_report.cc:383] FS layout report
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/data
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:04.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.443428 20519 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.444257 20519 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.444586 20519 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:04.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.445578 20519 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:04.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.447064 20519 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:59:04.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.447165 20519 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:59:04.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.447278 20519 ts_tablet_manager.cc:610] Registered 0 tablets
23:59:04.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.447335 20519 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:59:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.479117 20519 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.1:36839
23:59:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.479151 20651 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.1:36839 every 8 connection(s)
23:59:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.480428 20519 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.485934 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 20519
23:59:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.486147 20244 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal/instance
23:59:04.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.491123 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:0
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.499186 20654 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.499521 20654 heartbeater.cc:464] Registering TS with master...
23:59:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.500425 20654 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.503038 20357 ts_manager.cc:194] Registered new tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:04.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.507263 20653 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:04.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.507409 20653 heartbeater.cc:464] Registering TS with master...
23:59:04.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.507789 20653 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:04.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.509274 20286 ts_manager.cc:194] Registered new tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.510161 20652 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:04.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.510289 20652 heartbeater.cc:464] Registering TS with master...
23:59:04.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.510650 20652 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:04.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.511468 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.1:56429
23:59:04.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.512476 20428 ts_manager.cc:194] Registered new tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.657403 20483 catalog_manager.cc:1259] Loaded cluster ID: 06235fc4e144452eaeeea100ab14fdc2
23:59:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.657490 20483 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: loading cluster ID for follower catalog manager: success
23:59:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.659818 20483 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: acquiring CA information for follower catalog manager: success
23:59:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.661330 20483 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6640f710a7ce48eb96f01a08c4333169: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:59:04.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.821538 20660 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:04.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.821910 20660 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:04.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.822027 20660 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:04.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.829638 20660 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:04.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.829912 20660 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.2
23:59:04.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.839530 20660 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:0
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.841343 20660 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:04.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.843737 20660 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.857957 20668 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.858032 20667 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:04.858086 20670 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.858965 20660 server_base.cc:1034] running on GCE node
23:59:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.859642 20660 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:04.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.860646 20660 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:04.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.861924 20660 hybrid_clock.cc:648] HybridClock initialized: now 1783468744861829 us; error 59 us; skew 500 ppm
23:59:04.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.862322 20660 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:04.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.864912 20660 webserver.cc:460] Webserver started at http://127.19.197.2:33071/ using document root <none> and password file <none>
23:59:04.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.865897 20660 fs_manager.cc:359] Metadata directory not provided
23:59:04.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.866030 20660 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:04.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.866475 20660 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:59:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.869258 20660 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/instance:
23:59:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.870175 20660 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal/instance:
23:59:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.875396 20660 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.002s	sys 0.003s
23:59:04.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.878861 20676 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:04.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.880198 20660 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
23:59:04.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.880370 20660 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.880601 20660 fs_report.cc:383] FS layout report
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/data
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.896862 20660 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.897787 20660 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.898162 20660 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:04.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.899260 20660 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:04.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.900846 20660 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:59:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.900942 20660 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:59:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.901062 20660 ts_tablet_manager.cc:610] Registered 0 tablets
23:59:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.901119 20660 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:59:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.934406 20660 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.2:45931
23:59:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.934448 20788 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.2:45931 every 8 connection(s)
23:59:04.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.935721 20660 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:04.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.937464 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 20660
23:59:04.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.937637 20244 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal/instance
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.942200 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/logs
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.3:0
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.3
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.3
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:04.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.961303 20791 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:04.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.961584 20791 heartbeater.cc:464] Registering TS with master...
23:59:04.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.962178 20789 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:04.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.962342 20789 heartbeater.cc:464] Registering TS with master...
23:59:04.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.962502 20791 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:04.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.962806 20789 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.964815 20428 ts_manager.cc:194] Registered new tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.964901 20357 ts_manager.cc:194] Registered new tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.965559 20790 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.965687 20790 heartbeater.cc:464] Registering TS with master...
23:59:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.966162 20790 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:04.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.966998 20286 ts_manager.cc:194] Registered new tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:04.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:04.967861 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.2:38683
23:59:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.035517 20525 catalog_manager.cc:1259] Loaded cluster ID: 06235fc4e144452eaeeea100ab14fdc2
23:59:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.035653 20525 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a: loading cluster ID for follower catalog manager: success
23:59:05.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.039469 20525 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a: acquiring CA information for follower catalog manager: success
23:59:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.041791 20525 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5d231205e9a94388916806b12662111a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:59:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.278644 20796 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.278963 20796 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.279079 20796 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:05.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.286944 20796 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:05.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.287168 20796 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.3
23:59:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.296619 20796 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.3:0
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.3
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.3
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/logs
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.298278 20796 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.300108 20796 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.315666 20802 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:05.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.317139 20803 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:05.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.317765 20796 server_base.cc:1034] running on GCE node
23:59:05.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:05.318286 20805 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:05.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.319154 20796 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.320336 20796 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.321717 20796 hybrid_clock.cc:648] HybridClock initialized: now 1783468745321644 us; error 38 us; skew 500 ppm
23:59:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.322101 20796 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.324889 20796 webserver.cc:460] Webserver started at http://127.19.197.3:41827/ using document root <none> and password file <none>
23:59:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.325770 20796 fs_manager.cc:359] Metadata directory not provided
23:59:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.325913 20796 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:05.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.326304 20796 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:59:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.329975 20796 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/instance:
23:59:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67bf7defb0654350846098210079d01d"
23:59:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:05 on dist-test-slave-x4hg"
23:59:05.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.331014 20796 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal/instance:
23:59:05.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67bf7defb0654350846098210079d01d"
23:59:05.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:05 on dist-test-slave-x4hg"
23:59:05.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.337744 20796 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.000s
23:59:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.342115 20811 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:05.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.343775 20796 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.344075 20796 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67bf7defb0654350846098210079d01d"
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:05 on dist-test-slave-x4hg"
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.344310 20796 fs_report.cc:383] FS layout report
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/data
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:05.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.386032 20796 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.387080 20796 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.387403 20796 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:05.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.388667 20796 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.390807 20796 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:59:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.390918 20796 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:59:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.391031 20796 ts_tablet_manager.cc:610] Registered 0 tablets
23:59:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.391078 20796 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.438665 20796 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.3:33657
23:59:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.438764 20923 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.3:33657 every 8 connection(s)
23:59:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.439958 20796 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.445684 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 20796
23:59:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.445874 20244 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal/instance
23:59:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.458178 20925 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.458539 20925 heartbeater.cc:464] Registering TS with master...
23:59:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.458834 20924 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.458935 20924 heartbeater.cc:464] Registering TS with master...
23:59:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.459587 20926 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.459649 20924 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.459719 20925 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.459734 20926 heartbeater.cc:464] Registering TS with master...
23:59:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.460237 20926 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.461140 20286 ts_manager.cc:194] Registered new tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.461252 20357 ts_manager.cc:194] Registered new tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.461253 20428 ts_manager.cc:194] Registered new tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.462102 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.3:60125
23:59:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.469930 20244 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
23:59:05.471 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:59:05.472 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

23:59:05.484 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.19.197.62"
      port: 35395
    }
    bound_http_address {
      host: "127.19.197.62"
      port: 35969
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.19.197.61"
      port: 35737
    }
    bound_http_address {
      host: "127.19.197.61"
      port: 46569
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.19.197.60"
      port: 43759
    }
    bound_http_address {
      host: "127.19.197.60"
      port: 34589
    }
  }
}

23:59:05.489 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

23:59:05.492 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.19.197.1"
      port: 36839
    }
    bound_http_address {
      host: "127.19.197.1"
      port: 35163
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.19.197.2"
      port: 45931
    }
    bound_http_address {
      host: "127.19.197.2"
      port: 33071
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.19.197.3"
      port: 33657
    }
    bound_http_address {
      host: "127.19.197.3"
      port: 41827
    }
  }
}

23:59:05.492 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:59:05.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.515496 20653 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:05.680 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
23:59:05.870 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.197.61:35737(127.19.197.61:35737)] connecting to peer
23:59:05.925 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.197.60:43759(127.19.197.60:43759)] connecting to peer
23:59:05.926 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.197.62:35395(127.19.197.62:35395)] connecting to peer
23:59:05.964 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.197.61:35737(127.19.197.61:35737)] Successfully connected to peer
23:59:05.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:05.970467 20790 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:05.972 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.19.197.60:43759(127.19.197.60:43759)] Successfully connected to peer
23:59:06.036 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.197.62:35395(127.19.197.62:35395)] Successfully connected to peer
23:59:06.063 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.197.60
23:59:06.063 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.197.61
23:59:06.256 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.197.62
23:59:06.284 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xdfaeda06, L:/127.0.0.1:34586 - R:/127.19.197.60:43759] using SASL/PLAIN
23:59:06.284 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7c6fc145, L:/127.0.0.1:35290 - R:/127.19.197.61:35737] using SASL/PLAIN
23:59:06.312 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x815921bd, L:/127.0.0.1:46152 - R:/127.19.197.62:35395] using SASL/PLAIN
23:59:06.343 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
23:59:06.345 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=513), Traces: [0ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.197.61:35737, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.197.60:43759, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.197.62:35395, [465ms] Sub RPC ConnectToMaster: received response from server master-127.19.197.60:43759: OK, [473ms] Sub RPC ConnectToMaster: received response from server master-127.19.197.61:35737: OK, [484ms] Sub RPC ConnectToMaster: received response from server master-127.19.197.62:35395: OK, deferred=null) after lookup
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.402665 20286 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46152:
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820"
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:59:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
23:59:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:59:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:06.404809 20286 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
23:59:06.450 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 93bdf17f72e7453489684153a3a92eb6
23:59:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.463490 20587 tablet_service.cc:1467] Processing CreateTablet for tablet 444a22f6bc684186b6cc99f4d034b72b (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820 [id=93bdf17f72e7453489684153a3a92eb6]), partition=RANGE (key) PARTITION UNBOUNDED
23:59:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.464138 20925 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.464037 20724 tablet_service.cc:1467] Processing CreateTablet for tablet 444a22f6bc684186b6cc99f4d034b72b (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820 [id=93bdf17f72e7453489684153a3a92eb6]), partition=RANGE (key) PARTITION UNBOUNDED
23:59:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.465006 20587 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 444a22f6bc684186b6cc99f4d034b72b. 1 dirs total, 0 dirs full, 0 dirs failed
23:59:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.465263 20724 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 444a22f6bc684186b6cc99f4d034b72b. 1 dirs total, 0 dirs full, 0 dirs failed
23:59:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.467170 20859 tablet_service.cc:1467] Processing CreateTablet for tablet 444a22f6bc684186b6cc99f4d034b72b (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820 [id=93bdf17f72e7453489684153a3a92eb6]), partition=RANGE (key) PARTITION UNBOUNDED
23:59:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.468264 20859 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 444a22f6bc684186b6cc99f4d034b72b. 1 dirs total, 0 dirs full, 0 dirs failed
23:59:06.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.478828 20954 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap starting.
23:59:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.480947 20955 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Bootstrap starting.
23:59:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.483403 20954 tablet_bootstrap.cc:654] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Neither blocks nor log segments found. Creating new log.
23:59:06.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.484344 20955 tablet_bootstrap.cc:654] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Neither blocks nor log segments found. Creating new log.
23:59:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.485073 20954 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Log is configured to *not* fsync() on all Append() calls
23:59:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.485636 20955 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Log is configured to *not* fsync() on all Append() calls
23:59:06.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.491554 20955 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: No bootstrap required, opened a new log
23:59:06.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.491963 20955 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Time spent bootstrapping tablet: real 0.011s	user 0.010s	sys 0.000s
23:59:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.503367 20955 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.503971 20955 raft_consensus.cc:374] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:59:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.504220 20955 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 67bf7defb0654350846098210079d01d, State: Initialized, Role: FOLLOWER
23:59:06.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.505048 20955 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.505236 20954 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: No bootstrap required, opened a new log
23:59:06.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.505578 20954 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent bootstrapping tablet: real 0.027s	user 0.011s	sys 0.011s
23:59:06.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.507331 20956 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap starting.
23:59:06.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.510864 20956 tablet_bootstrap.cc:654] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Neither blocks nor log segments found. Creating new log.
23:59:06.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.511838 20956 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Log is configured to *not* fsync() on all Append() calls
23:59:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.515836 20955 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Time spent starting tablet: real 0.024s	user 0.015s	sys 0.004s
23:59:06.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.518601 20954 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.519210 20954 raft_consensus.cc:374] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:59:06.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.519424 20954 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Initialized, Role: FOLLOWER
23:59:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.520180 20954 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.521600 20956 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: No bootstrap required, opened a new log
23:59:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.522006 20956 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent bootstrapping tablet: real 0.015s	user 0.005s	sys 0.005s
23:59:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.522727 20954 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent starting tablet: real 0.017s	user 0.014s	sys 0.000s
23:59:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.529902 20956 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.530297 20956 raft_consensus.cc:374] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:59:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.530362 20956 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Initialized, Role: FOLLOWER
23:59:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.530951 20956 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.532630 20956 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent starting tablet: real 0.010s	user 0.009s	sys 0.000s
23:59:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.681243 20962 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:59:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.681533 20962 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.683209 20962 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:06.690243 20792 tablet.cc:2369] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:59:06.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.692273 20879 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:06.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.692409 20607 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf" is_pre_election: true
23:59:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.692844 20879 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 0.
23:59:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.692931 20607 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 0.
23:59:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.693699 20679 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 
23:59:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.694144 20962 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:59:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.694285 20962 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:59:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.694380 20962 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 0 FOLLOWER]: Advancing to term 1
23:59:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:06.695060 20927 tablet.cc:2369] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:59:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.697324 20962 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.697880 20962 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 1 election: Requested vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.698288 20607 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.698352 20879 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d"
23:59:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.698493 20879 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 0 FOLLOWER]: Advancing to term 1
23:59:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.698489 20607 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 0 FOLLOWER]: Advancing to term 1
23:59:06.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.700331 20879 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 1.
23:59:06.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.700516 20607 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 1.
23:59:06.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.700659 20679 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 
23:59:06.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.700958 20962 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Leader election won for term 1
23:59:06.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.701467 20962 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 LEADER]: Becoming Leader. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Running, Role: LEADER
23:59:06.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.702208 20962 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:06.707265 20286 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 reported cstate change: term changed from 0 to 1, leader changed from <none> to df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2). New cstate: current_term: 1 leader_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: HEALTHY } } }
23:59:06.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:06.734099 20655 tablet.cc:2369] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:59:06.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 93bdf17f72e7453489684153a3a92eb6
23:59:06.979 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:06.986 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:06.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [10ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=null) after lookup
23:59:06.997 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:06.999 [DEBUG - kudu-nio-1] (Connection.java:757) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] Successfully connected to peer
23:59:07.046 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x570e3495, L:/127.0.0.1:33934 - R:/127.19.197.2:45931] using TOKEN/null
23:59:07.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:07.055369 20879 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 1 FOLLOWER]: Refusing update from remote peer df2cc700bf114fb3af3f4fb7fdd3b1b0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:59:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:07.055423 20607 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 1 FOLLOWER]: Refusing update from remote peer df2cc700bf114fb3af3f4fb7fdd3b1b0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:59:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:07.056370 20965 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:59:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:07.056716 20962 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:59:07.151 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:07.153 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.2:45931
23:59:07.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:07.155628 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 20660
23:59:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:07.173723 20784 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:07.272089 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 20660
23:59:07.290 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:07.294 [DEBUG - kudu-nio-1] (Connection.java:239) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] handling channelInactive
23:59:07.295 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state READY due to: connection closed
23:59:07.296 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: connection closed
23:59:07.315 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:07.316 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:07.316 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [6ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: connection closed, [6ms] delaying RPC due to: Network error: connection closed, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [25ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1880887576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@76a558c6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@568d5e71 -> wakeup thread Time-limited test)) after lookup
23:59:07.317 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:07.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.318 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.335 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:07.336 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:07.336 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [6ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: connection closed, [6ms] delaying RPC due to: Network error: connection closed, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [25ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [27ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [28ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [28ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [45ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1880887576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@76a558c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@568d5e71 -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936))) after lookup
23:59:07.337 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:07.339 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.339 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.356 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:07.356 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:07.357 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [6ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: connection closed, [6ms] delaying RPC due to: Network error: connection closed, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [25ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [27ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [28ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [28ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [45ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [47ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [49ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [49ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [65ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1880887576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@76a558c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@568d5e71 -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208))) after lookup
23:59:07.358 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:07.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.359 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.395 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:07.396 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:07.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=105), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [6ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: connection closed, [6ms] delaying RPC due to: Network error: connection closed, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [25ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [27ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [28ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [28ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [45ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [47ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [49ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [49ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [65ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [67ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [69ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [102ms] refreshing cache from master, [102ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [105ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1880887576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@76a558c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208) -> (continuation of Deferred@246903758 after retry RPC@1264486050), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@568d5e71 -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208) -> (continuation of Deferred@246903758 after retry RPC@1264486050))) after lookup
23:59:07.398 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:07.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.400 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.415 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:07.415 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:07.416 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=125), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [6ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: connection closed, [6ms] delaying RPC due to: Network error: connection closed, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [25ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [27ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [28ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [28ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [45ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [47ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [49ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [49ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [65ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [67ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [69ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [102ms] refreshing cache from master, [102ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [105ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [107ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [110ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [110ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [121ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [124ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1880887576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@76a558c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208) -> (continuation of Deferred@246903758 after retry RPC@1264486050) -> (continuation of Deferred@1373466546 after retry RPC@1854511293), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@568d5e71 -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208) -> (continuation of Deferred@246903758 after retry RPC@1264486050) -> (continuation of Deferred@1373466546 after retry RPC@1854511293))) after lookup
23:59:07.417 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:07.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:07.418 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:08.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.611722 20983 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 1 FOLLOWER]: Starting pre-election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.611899 20983 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.612839 20984 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.612985 20984 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.613498 20983 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.620457 20984 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:08.624578 20542 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:08.625994 20814 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.626402 20879 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.626654 20879 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 1.
23:59:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:08.627245 20542 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.627700 20542 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.628129 20983 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 1 FOLLOWER]: Leader pre-election won for term 2
23:59:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.628266 20983 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 1 FOLLOWER]: Starting leader election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.628364 20983 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 1 FOLLOWER]: Advancing to term 2
23:59:08.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.630646 20983 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:08.631029 20814 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.631268 20983 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 2 election: Requested vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.631635 20879 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d"
23:59:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.631830 20879 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 1 FOLLOWER]: Advancing to term 2
23:59:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.631961 20607 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf" is_pre_election: true
23:59:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.632247 20607 raft_consensus.cc:2379] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 67bf7defb0654350846098210079d01d in current term 2: Already voted for candidate 6a4687af738441b284ae2313d7276cbf in this term.
23:59:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.633028 20814 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 67bf7defb0654350846098210079d01d; no voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:08.633716 20542 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.634596 20879 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 2.
23:59:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.634794 20984 raft_consensus.cc:2726] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:59:08.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.634977 20542 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:08.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.635246 20983 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Leader election won for term 2
23:59:08.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.635653 20983 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 LEADER]: Becoming Leader. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Running, Role: LEADER
23:59:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.636121 20983 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:08.639528 20285 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf reported cstate change: term changed from 1 to 2, leader changed from df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2) to 6a4687af738441b284ae2313d7276cbf (127.19.197.1). New cstate: current_term: 2 leader_uuid: "6a4687af738441b284ae2313d7276cbf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: UNKNOWN } } }
23:59:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.198261 20879 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Refusing update from remote peer 6a4687af738441b284ae2313d7276cbf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
23:59:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.199059 21006 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Connected to new peer: Peer: permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
23:59:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.206271 20542 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf -> Peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Couldn't send request to peer df2cc700bf114fb3af3f4fb7fdd3b1b0. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:09.376 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:09.377 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:09.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2087), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [6ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: connection closed, [6ms] delaying RPC due to: Network error: connection closed, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [25ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [27ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [28ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [28ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [45ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [47ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [49ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [49ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [65ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [67ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [69ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [102ms] refreshing cache from master, [102ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [105ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [107ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [110ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [110ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [121ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [124ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [127ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [128ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [128ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2082ms] refreshing cache from master, [2082ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [2086ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1880887576(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@76a558c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208) -> (continuation of Deferred@246903758 after retry RPC@1264486050) -> (continuation of Deferred@1373466546 after retry RPC@1854511293) -> (continuation of Deferred@847459288 after retry RPC@851738488), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@568d5e71 -> wakeup thread Time-limited test -> (continuation of Deferred@823519318 after retry RPC@476569936) -> (continuation of Deferred@722640467 after retry RPC@148670208) -> (continuation of Deferred@246903758 after retry RPC@1264486050) -> (continuation of Deferred@1373466546 after retry RPC@1854511293) -> (continuation of Deferred@847459288 after retry RPC@851738488))) after lookup
23:59:09.379 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:09.381 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] Successfully connected to peer
23:59:09.399 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe3d53350, L:/127.0.0.1:34530 - R:/127.19.197.1:36839] using TOKEN/null
23:59:09.421 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.2:45931
23:59:09.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.423672 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.750445 21018 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:09.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.750743 21018 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:09.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.750860 21018 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:09.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.758008 21018 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:09.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.758289 21018 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.2
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.769994 21018 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.771376 21018 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.773047 21018 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.785926 21023 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.785877 21026 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:09.785959 21024 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.786947 21018 server_base.cc:1034] running on GCE node
23:59:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.787722 21018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.788630 21018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.789928 21018 hybrid_clock.cc:648] HybridClock initialized: now 1783468749789807 us; error 48 us; skew 500 ppm
23:59:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.790287 21018 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.792591 21018 webserver.cc:460] Webserver started at http://127.19.197.2:33071/ using document root <none> and password file <none>
23:59:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.793354 21018 fs_manager.cc:359] Metadata directory not provided
23:59:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.793515 21018 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:09.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.798815 21018 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
23:59:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.801635 21032 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:09.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.802985 21018 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.803170 21018 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.804145 21018 fs_report.cc:383] FS layout report
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/data
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.817397 21018 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.818367 21018 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.818693 21018 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.819651 21018 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.821535 21039 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:09.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.827308 21018 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:09.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.827440 21018 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.000s	sys 0.000s
23:59:09.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.827564 21018 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.830637 21018 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.830744 21018 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
23:59:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.831048 21039 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap starting.
23:59:09.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.853811 21039 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Log is configured to *not* fsync() on all Append() calls
23:59:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.869735 21039 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.870798 21039 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap complete.
23:59:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.871783 21018 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.2:45931
23:59:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.871968 21039 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent bootstrapping tablet: real 0.041s	user 0.022s	sys 0.016s
23:59:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.871807 21146 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.2:45931 every 8 connection(s)
23:59:09.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.873186 21018 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.879462 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 21018
23:59:09.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.883515 21039 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.885334 21039 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Initialized, Role: FOLLOWER
23:59:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.886221 21039 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.894462 21039 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent starting tablet: real 0.022s	user 0.018s	sys 0.004s
23:59:09.915 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:09.916 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.1:36839
23:59:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.916639 21148 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.916978 21148 heartbeater.cc:464] Registering TS with master...
23:59:09.917 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.917873 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 20519
23:59:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.917881 21148 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.918452 21149 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.918598 21149 heartbeater.cc:464] Registering TS with master...
23:59:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.919093 21149 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.921113 20357 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.922284 20285 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.923468 20285 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.2:42537
23:59:09.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.927606 21148 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:09.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.937410 21147 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:09.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.937583 21147 heartbeater.cc:464] Registering TS with master...
23:59:09.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.937934 21147 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:09.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.938408 20647 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:09.939394 20428 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:10.036720 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 20519
23:59:10.056 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] handling channelInactive
23:59:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:10.055770 20815 connection.cc:535] server connection from 127.19.197.1:44317 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:59:10.056 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state READY due to: connection closed
23:59:10.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:10.057 [DEBUG - Time-limited test] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:10.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.058 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.075 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:10.075 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:10.076 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [18ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1125602828(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@17e4c9ff -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f2d20cf -> wakeup thread Time-limited test)) after lookup
23:59:10.076 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:10.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.078 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.096 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:10.096 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:10.098 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [18ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [20ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [22ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [39ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1125602828(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@17e4c9ff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f2d20cf -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262))) after lookup
23:59:10.098 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:10.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.100 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.100 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.136 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:10.136 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:10.137 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [18ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [20ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [22ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [39ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [44ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [79ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1125602828(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@17e4c9ff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f2d20cf -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520))) after lookup
23:59:10.137 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:10.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.139 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.255 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:10.255 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:10.256 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=199), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [18ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [20ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [22ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [39ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [44ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [79ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [81ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [83ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [195ms] refreshing cache from master, [196ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [198ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1125602828(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@17e4c9ff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520) -> (continuation of Deferred@157717025 after retry RPC@1364227244), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f2d20cf -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520) -> (continuation of Deferred@157717025 after retry RPC@1364227244))) after lookup
23:59:10.257 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:10.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.258 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.596 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:10.597 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:10.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=541), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [18ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [20ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [22ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [39ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [44ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [79ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [81ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [83ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [195ms] refreshing cache from master, [196ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [198ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [201ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [202ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [202ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [537ms] refreshing cache from master, [537ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [540ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1125602828(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@17e4c9ff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520) -> (continuation of Deferred@157717025 after retry RPC@1364227244) -> (continuation of Deferred@876068242 after retry RPC@686590021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f2d20cf -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520) -> (continuation of Deferred@157717025 after retry RPC@1364227244) -> (continuation of Deferred@876068242 after retry RPC@686590021))) after lookup
23:59:10.599 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:10.601 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.601 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:10.601 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.366039 21162 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:59:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.366329 21162 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.367820 21162 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:11.369419 21035 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:11.370976 21035 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.374472 20879 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.375092 21035 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf
23:59:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.375402 21162 raft_consensus.cc:2726] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:59:11.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.538406 21167 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:11.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.538583 21167 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:11.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.539103 21167 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:11.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:11.541703 20814 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:11.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.547576 21102 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" is_pre_election: true
23:59:11.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.548074 21102 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 67bf7defb0654350846098210079d01d in term 1.
23:59:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.549137 20814 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.549431 21167 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Leader pre-election won for term 3
23:59:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.549542 21167 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Starting leader election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.549607 21167 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 2 FOLLOWER]: Advancing to term 3
23:59:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.551892 21167 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.552368 21167 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 3 election: Requested vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.553165 21102 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.553381 21102 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 1 FOLLOWER]: Advancing to term 3
23:59:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:11.553869 20814 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.557422 21102 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 67bf7defb0654350846098210079d01d in term 3.
23:59:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.557838 20814 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.558143 21167 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 3 FOLLOWER]: Leader election won for term 3
23:59:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.558727 21167 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 3 LEADER]: Becoming Leader. State: Replica: 67bf7defb0654350846098210079d01d, State: Running, Role: LEADER
23:59:11.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.559249 21167 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:11.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.563019 20285 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d reported cstate change: term changed from 2 to 3, leader changed from 6a4687af738441b284ae2313d7276cbf (127.19.197.1) to 67bf7defb0654350846098210079d01d (127.19.197.3). New cstate: current_term: 3 leader_uuid: "67bf7defb0654350846098210079d01d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: UNKNOWN } } }
23:59:11.876 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:11.877 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:11.878 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1820), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [18ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [20ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [22ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [39ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [44ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [79ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [81ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [83ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [195ms] refreshing cache from master, [196ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [198ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [201ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [202ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [202ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [537ms] refreshing cache from master, [537ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [540ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [542ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [545ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [545ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [1816ms] refreshing cache from master, [1816ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1819ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1125602828(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@17e4c9ff -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520) -> (continuation of Deferred@157717025 after retry RPC@1364227244) -> (continuation of Deferred@876068242 after retry RPC@686590021) -> (continuation of Deferred@2664828 after retry RPC@1578979608), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f2d20cf -> wakeup thread Time-limited test -> (continuation of Deferred@984074069 after retry RPC@213562262) -> (continuation of Deferred@1988603753 after retry RPC@1655134520) -> (continuation of Deferred@157717025 after retry RPC@1364227244) -> (continuation of Deferred@876068242 after retry RPC@686590021) -> (continuation of Deferred@2664828 after retry RPC@1578979608))) after lookup
23:59:11.879 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:11.880 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] Successfully connected to peer
23:59:11.901 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd14480fb, L:/127.0.0.1:53378 - R:/127.19.197.3:33657] using TOKEN/null
23:59:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.906769 21102 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 3 FOLLOWER]: Refusing update from remote peer 67bf7defb0654350846098210079d01d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
23:59:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:11.907111 20814 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d -> Peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Couldn't send request to peer 6a4687af738441b284ae2313d7276cbf. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.907685 21172 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [LEADER]: Connected to new peer: Peer: permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
23:59:11.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.921783 21185 mvcc.cc:205] Tried to move back new op lower bound from 7305088007800881152 to 7305088006390161408. Current Snapshot: MvccSnapshot[applied={T|T < 7305088007800881152}]
23:59:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.927428 21187 mvcc.cc:205] Tried to move back new op lower bound from 7305088007800881152 to 7305088006390161408. Current Snapshot: MvccSnapshot[applied={T|T < 7305088007800881152}]
23:59:11.947 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.1:36839
23:59:11.948 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:11.949083 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.281780 21194 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.282110 21194 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.282213 21194 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.289332 21194 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.289487 21194 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.1
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.298684 21194 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.299896 21194 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:12.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.301533 21194 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.314531 21199 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.314582 21202 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:12.314504 21200 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.315447 21194 server_base.cc:1034] running on GCE node
23:59:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.316068 21194 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.316962 21194 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.318176 21194 hybrid_clock.cc:648] HybridClock initialized: now 1783468752318086 us; error 45 us; skew 500 ppm
23:59:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.318491 21194 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:12.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.323410 21194 webserver.cc:460] Webserver started at http://127.19.197.1:35163/ using document root <none> and password file <none>
23:59:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.323994 21194 fs_manager.cc:359] Metadata directory not provided
23:59:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.324101 21194 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.328727 21194 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
23:59:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.331488 21208 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:12.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.332772 21194 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
23:59:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.332934 21194 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.333864 21194 fs_report.cc:383] FS layout report
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/data
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.346220 21194 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:12.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.346982 21194 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:12.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.347194 21194 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.348232 21194 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:12.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.350335 21215 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:12.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.355502 21194 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:12.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.355593 21194 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:12.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.355674 21194 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.358439 21194 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.358507 21194 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
23:59:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.358881 21215 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap starting.
23:59:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.382316 21215 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Log is configured to *not* fsync() on all Append() calls
23:59:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.399545 21194 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.1:36839
23:59:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.399592 21322 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.1:36839 every 8 connection(s)
23:59:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.400715 21194 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.400864 21215 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.401865 21215 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap complete.
23:59:12.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.401998 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 21194
23:59:12.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:12.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.403594 21215 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent bootstrapping tablet: real 0.045s	user 0.023s	sys 0.019s
23:59:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.411809 21215 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.413110 21215 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Initialized, Role: FOLLOWER
23:59:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.413977 21215 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:12.421 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:12.425 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.3:33657
23:59:12.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:59:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.425846 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 20796
23:59:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.428098 21215 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent starting tablet: real 0.024s	user 0.011s	sys 0.011s
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.438735 21323 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.438762 21325 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.439078 21323 heartbeater.cc:464] Registering TS with master...
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.439083 21325 heartbeater.cc:464] Registering TS with master...
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.440088 21325 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.440116 21323 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.441133 21324 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.441244 21324 heartbeater.cc:464] Registering TS with master...
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.441668 21324 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.442821 20428 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.442847 20285 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:12.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.444335 20285 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.1:47737
23:59:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.445317 20357 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.447540 21324 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:12.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.452872 20919 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:12.549044 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 20796
23:59:12.567 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] handling channelInactive
23:59:12.567 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state READY due to: connection closed
23:59:12.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:12.568 [DEBUG - Time-limited test] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:12.570 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.570 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.595 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:12.595 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:12.595 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [26ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@888698459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@b3a8293 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@844bdc6 -> wakeup thread Time-limited test)) after lookup
23:59:12.597 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:12.598 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.598 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.598 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.615 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:12.616 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:12.616 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [26ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server 67bf7defb0654350846098210079d01d, [30ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [30ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [47ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@888698459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@b3a8293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@844bdc6 -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390))) after lookup
23:59:12.617 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:12.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.618 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.655 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:12.655 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:12.655 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [26ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server 67bf7defb0654350846098210079d01d, [30ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [30ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [47ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [48ms] sending RPC to server 67bf7defb0654350846098210079d01d, [50ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [50ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [86ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@888698459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@b3a8293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@844bdc6 -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768))) after lookup
23:59:12.656 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:12.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.657 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.674 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:12.675 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:12.675 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=107), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [26ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server 67bf7defb0654350846098210079d01d, [30ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [30ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [47ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [48ms] sending RPC to server 67bf7defb0654350846098210079d01d, [50ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [50ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [86ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [88ms] sending RPC to server 67bf7defb0654350846098210079d01d, [89ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [89ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [106ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@888698459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@b3a8293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@844bdc6 -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822))) after lookup
23:59:12.676 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:12.677 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.677 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.677 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.955 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:12.955 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:12.956 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=387), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [26ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server 67bf7defb0654350846098210079d01d, [30ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [30ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [47ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [48ms] sending RPC to server 67bf7defb0654350846098210079d01d, [50ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [50ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [86ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [88ms] sending RPC to server 67bf7defb0654350846098210079d01d, [89ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [89ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [106ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [107ms] sending RPC to server 67bf7defb0654350846098210079d01d, [109ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [109ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [384ms] refreshing cache from master, [384ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [387ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@888698459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@b3a8293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822) -> (continuation of Deferred@776524671 after retry RPC@893136654), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@844bdc6 -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822) -> (continuation of Deferred@776524671 after retry RPC@893136654))) after lookup
23:59:12.957 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:12.957 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.957 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:12.957 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:13.036 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:13.036 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:13.037 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=468), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [26ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server 67bf7defb0654350846098210079d01d, [30ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [30ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [47ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [48ms] sending RPC to server 67bf7defb0654350846098210079d01d, [50ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [50ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [86ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [88ms] sending RPC to server 67bf7defb0654350846098210079d01d, [89ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [89ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [106ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [107ms] sending RPC to server 67bf7defb0654350846098210079d01d, [109ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [109ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [384ms] refreshing cache from master, [384ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [387ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [388ms] sending RPC to server 67bf7defb0654350846098210079d01d, [389ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [390ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [465ms] refreshing cache from master, [465ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [467ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@888698459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@b3a8293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822) -> (continuation of Deferred@776524671 after retry RPC@893136654) -> (continuation of Deferred@145485946 after retry RPC@654615016), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@844bdc6 -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822) -> (continuation of Deferred@776524671 after retry RPC@893136654) -> (continuation of Deferred@145485946 after retry RPC@654615016))) after lookup
23:59:13.037 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:13.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:13.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:13.039 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:13.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:13.625759 21338 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:59:13.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:13.626012 21338 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:13.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:13.627569 21338 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:13.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:13.629314 21211 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:13.630975 21211 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:13.634812 21102 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" is_pre_election: true
23:59:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:13.635591 21211 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a4687af738441b284ae2313d7276cbf; no voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:13.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:13.635926 21338 raft_consensus.cc:2726] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
23:59:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.050127 21343 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 67bf7defb0654350846098210079d01d)
23:59:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.050354 21343 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:14.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.051066 21343 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:14.055234 21035 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.059006 21277 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf" is_pre_election: true
23:59:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.059444 21277 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 2.
23:59:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.060421 21035 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 67bf7defb0654350846098210079d01d
23:59:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.060707 21343 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 3 FOLLOWER]: Leader pre-election won for term 4
23:59:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.060807 21343 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 67bf7defb0654350846098210079d01d)
23:59:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.060865 21343 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 3 FOLLOWER]: Advancing to term 4
23:59:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.062793 21343 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.063269 21343 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 4 election: Requested vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.063743 21277 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.063951 21277 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 2 FOLLOWER]: Advancing to term 4
23:59:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:14.064808 21035 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:14.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.067780 21277 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 4.
23:59:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.068154 21035 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 67bf7defb0654350846098210079d01d
23:59:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.068423 21343 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Leader election won for term 4
23:59:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.068809 21343 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 LEADER]: Becoming Leader. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Running, Role: LEADER
23:59:14.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.069240 21343 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:14.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.072736 20285 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 reported cstate change: term changed from 3 to 4, leader changed from 67bf7defb0654350846098210079d01d (127.19.197.3) to df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2). New cstate: current_term: 4 leader_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: HEALTHY } } }
23:59:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:14.544992 21035 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 -> Peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Couldn't send request to peer 67bf7defb0654350846098210079d01d. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:14.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.601773 21277 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 4 FOLLOWER]: Refusing update from remote peer df2cc700bf114fb3af3f4fb7fdd3b1b0: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
23:59:14.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:14.602638 21343 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
23:59:15.855 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:15.855 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:15.856 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3288), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [26ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server 67bf7defb0654350846098210079d01d, [30ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [30ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [47ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [48ms] sending RPC to server 67bf7defb0654350846098210079d01d, [50ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [50ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [86ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [88ms] sending RPC to server 67bf7defb0654350846098210079d01d, [89ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [89ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [106ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [107ms] sending RPC to server 67bf7defb0654350846098210079d01d, [109ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [109ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [384ms] refreshing cache from master, [384ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [387ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [388ms] sending RPC to server 67bf7defb0654350846098210079d01d, [389ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [390ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [465ms] refreshing cache from master, [465ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [467ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [469ms] sending RPC to server 67bf7defb0654350846098210079d01d, [471ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [472ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [3284ms] refreshing cache from master, [3284ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [3286ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@888698459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@b3a8293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822) -> (continuation of Deferred@776524671 after retry RPC@893136654) -> (continuation of Deferred@145485946 after retry RPC@654615016) -> (continuation of Deferred@607510383 after retry RPC@649867744), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@844bdc6 -> wakeup thread Time-limited test -> (continuation of Deferred@1220204224 after retry RPC@583996390) -> (continuation of Deferred@677642129 after retry RPC@1927911768) -> (continuation of Deferred@810606598 after retry RPC@1281029822) -> (continuation of Deferred@776524671 after retry RPC@893136654) -> (continuation of Deferred@145485946 after retry RPC@654615016) -> (continuation of Deferred@607510383 after retry RPC@649867744))) after lookup
23:59:15.857 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:15.857 [DEBUG - kudu-nio-0] (Connection.java:757) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] Successfully connected to peer
23:59:15.880 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x38bfbf31, L:/127.0.0.1:55452 - R:/127.19.197.2:45931] using TOKEN/null
23:59:15.922 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.3:33657
23:59:15.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:15.924098 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/logs
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.3:33657
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.3
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.3
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.306538 21382 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.307111 21382 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.307277 21382 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.315023 21382 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.315274 21382 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.3
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.326004 21382 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.3:33657
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.3
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.3
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/logs
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:16.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.327888 21382 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:16.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.329747 21382 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:16.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.344036 21387 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:16.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.343992 21390 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.344069 21388 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.345297 21382 server_base.cc:1034] running on GCE node
23:59:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.345922 21382 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:16.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.347012 21382 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.348284 21382 hybrid_clock.cc:648] HybridClock initialized: now 1783468756348162 us; error 57 us; skew 500 ppm
23:59:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.348601 21382 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:16.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.351033 21382 webserver.cc:460] Webserver started at http://127.19.197.3:41827/ using document root <none> and password file <none>
23:59:16.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.351712 21382 fs_manager.cc:359] Metadata directory not provided
23:59:16.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.351887 21382 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.356858 21382 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.002s
23:59:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.359561 21396 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.360739 21382 fs_manager.cc:725] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
23:59:16.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.360935 21382 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:16.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67bf7defb0654350846098210079d01d"
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:05 on dist-test-slave-x4hg"
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.361886 21382 fs_report.cc:383] FS layout report
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/data
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:16.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.377586 21382 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:16.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.378577 21382 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:16.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.378948 21382 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.380164 21382 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.382439 21403 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:16.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.387725 21382 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.387869 21382 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.000s	sys 0.001s
23:59:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.387998 21382 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:16.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.391112 21382 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:16.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.391216 21382 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.001s	sys 0.002s
23:59:16.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.391479 21403 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Bootstrap starting.
23:59:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.413913 21403 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Log is configured to *not* fsync() on all Append() calls
23:59:16.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.439424 21382 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.3:33657
23:59:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.439463 21510 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.3:33657 every 8 connection(s)
23:59:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.440021 21403 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.440637 21403 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Bootstrap complete.
23:59:16.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.441056 21382 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:16.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.442086 21403 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Time spent bootstrapping tablet: real 0.051s	user 0.033s	sys 0.010s
23:59:16.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.446297 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 21382
23:59:16.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.456634 21403 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.458455 21403 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 67bf7defb0654350846098210079d01d, State: Initialized, Role: FOLLOWER
23:59:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.459302 21403 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:16.495 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:16.502 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.2:45931
23:59:16.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.503082 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 21018
23:59:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.508464 21403 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Time spent starting tablet: real 0.066s	user 0.044s	sys 0.019s
23:59:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.517336 21511 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.517640 21511 heartbeater.cc:464] Registering TS with master...
23:59:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.518689 21511 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:16.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.522394 20427 ts_manager.cc:194] Re-registered known tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.527631 21512 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.527747 21512 heartbeater.cc:464] Registering TS with master...
23:59:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.528193 21512 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.528970 21513 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.529071 21513 heartbeater.cc:464] Registering TS with master...
23:59:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.529440 21513 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.530910 20286 ts_manager.cc:194] Re-registered known tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.531005 20356 ts_manager.cc:194] Re-registered known tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.532382 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.3:56281
23:59:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.535826 21142 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.536139 21512 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:16.641278 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 21018
23:59:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.659482 20414 connection.cc:535] server connection from 127.19.197.2:59407 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:59:16.660 [DEBUG - kudu-nio-0] (Connection.java:239) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] handling channelInactive
23:59:16.660 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state READY due to: connection closed
23:59:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:16.660676 21527 negotiation.cc:337] Failed RPC negotiation. Trace:
23:59:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:16.639144 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.19.197.2:55351 (local address 127.19.197.3:33657)
23:59:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:16.642666 (+  3522us) server_negotiation.cc:206] Beginning negotiation
23:59:16.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:16.642675 (+     9us) server_negotiation.cc:399] Waiting for connection header
23:59:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:16.659620 (+ 16945us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.197.2:55351: BlockingRecv error: recv got EOF from 127.19.197.2:55351 (error 108)
23:59:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3353,"thread_start_us":3257,"threads_started":1}
23:59:16.661 [DEBUG - Time-limited test] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:16.663 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.663 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.663 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.675 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:16.675 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:16.676 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [14ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@948360340(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@134ab3ba -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25934d3a -> wakeup thread Time-limited test)) after lookup
23:59:16.676 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:16.677 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.677 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.677 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.695 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:16.695 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:16.696 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [14ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [15ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [16ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [16ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [33ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@948360340(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@134ab3ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25934d3a -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915))) after lookup
23:59:16.696 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:16.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.698 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.735 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:16.735 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:16.736 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [14ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [15ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [16ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [16ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [33ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [35ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [37ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [38ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [73ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@948360340(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@134ab3ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25934d3a -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563))) after lookup
23:59:16.737 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:16.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.738 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.855 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:16.855 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:16.856 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=194), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [14ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [15ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [16ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [16ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [33ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [35ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [37ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [38ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [73ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [75ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [77ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [77ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [194ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@948360340(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@134ab3ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25934d3a -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336))) after lookup
23:59:16.857 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:16.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:16.859 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:17.215 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:17.216 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:17.216 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=554), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [14ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [15ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [16ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [16ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [33ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [35ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [37ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [38ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [73ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [75ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [77ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [77ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [194ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [195ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [197ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [198ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [554ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@948360340(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@134ab3ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336) -> (continuation of Deferred@1179671083 after retry RPC@2048262914), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25934d3a -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336) -> (continuation of Deferred@1179671083 after retry RPC@2048262914))) after lookup
23:59:17.217 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:17.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:17.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:17.218 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:17.895 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:17.896 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:17.896 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1235), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [14ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [15ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [16ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [16ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [33ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [35ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [37ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [38ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [73ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [75ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [77ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [77ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [194ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [195ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [197ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [198ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [554ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [555ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [557ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [557ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1231ms] refreshing cache from master, [1231ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1234ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@948360340(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@134ab3ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336) -> (continuation of Deferred@1179671083 after retry RPC@2048262914) -> (continuation of Deferred@1998020838 after retry RPC@1809071731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25934d3a -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336) -> (continuation of Deferred@1179671083 after retry RPC@2048262914) -> (continuation of Deferred@1998020838 after retry RPC@1809071731))) after lookup
23:59:17.897 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:17.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:17.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:17.899 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.969627 21528 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 4 FOLLOWER]: Starting pre-election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:17.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.969883 21528 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:17.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.970575 21528 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:17.973323 21211 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.977972 21466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.978457 21466 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 3.
23:59:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.979409 21211 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.979708 21528 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 4 FOLLOWER]: Leader pre-election won for term 5
23:59:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.979807 21528 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 4 FOLLOWER]: Starting leader election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.979864 21528 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 4 FOLLOWER]: Advancing to term 5
23:59:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.981737 21528 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.982275 21528 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 5 election: Requested vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.982718 21466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d"
23:59:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.982929 21466 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 3 FOLLOWER]: Advancing to term 5
23:59:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:17.983752 21211 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:17.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.986896 21466 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 5.
23:59:17.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.987450 21211 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:17.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.987694 21528 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Leader election won for term 5
23:59:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.988078 21528 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 LEADER]: Becoming Leader. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Running, Role: LEADER
23:59:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.988576 21528 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:17.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:17.991735 20286 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf reported cstate change: term changed from 4 to 5, leader changed from df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2) to 6a4687af738441b284ae2313d7276cbf (127.19.197.1). New cstate: current_term: 5 leader_uuid: "6a4687af738441b284ae2313d7276cbf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: UNKNOWN } } }
23:59:18.175 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:18.175 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:18.175 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1514), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [14ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [15ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [16ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [16ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [33ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [35ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [37ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [38ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [73ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [75ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [77ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [77ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [191ms] refreshing cache from master, [191ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [194ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [195ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [197ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [198ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [551ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [554ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [555ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [557ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [557ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1231ms] refreshing cache from master, [1231ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1234ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [1236ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [1238ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1238ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1510ms] refreshing cache from master, [1511ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1513ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@948360340(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@134ab3ba -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336) -> (continuation of Deferred@1179671083 after retry RPC@2048262914) -> (continuation of Deferred@1998020838 after retry RPC@1809071731) -> (continuation of Deferred@1301604546 after retry RPC@125395109), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25934d3a -> wakeup thread Time-limited test -> (continuation of Deferred@188512198 after retry RPC@136536915) -> (continuation of Deferred@1962822476 after retry RPC@1741981563) -> (continuation of Deferred@1380037161 after retry RPC@1059093336) -> (continuation of Deferred@1179671083 after retry RPC@2048262914) -> (continuation of Deferred@1998020838 after retry RPC@1809071731) -> (continuation of Deferred@1301604546 after retry RPC@125395109))) after lookup
23:59:18.177 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:18.178 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] Successfully connected to peer
23:59:18.198 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3647647f, L:/127.0.0.1:54382 - R:/127.19.197.1:36839] using TOKEN/null
23:59:18.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.204538 21211 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf -> Peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Couldn't send request to peer df2cc700bf114fb3af3f4fb7fdd3b1b0. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:18.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.204650 21466 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 5 FOLLOWER]: Refusing update from remote peer 6a4687af738441b284ae2313d7276cbf: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 18. (index mismatch)
23:59:18.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.205449 21533 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Connected to new peer: Peer: permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
23:59:18.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.214143 21547 mvcc.cc:205] Tried to move back new op lower bound from 7305088033595748352 to 7305088032724140032. Current Snapshot: MvccSnapshot[applied={T|T < 7305088033595748352}]
23:59:18.233 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.2:45931
23:59:18.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.234455 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.577971 21556 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.578323 21556 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.578408 21556 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:18.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.585661 21556 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:18.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.585812 21556 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.2
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.595136 21556 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.596388 21556 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.598215 21556 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.611166 21562 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.611131 21565 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.611091 21563 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.611999 21556 server_base.cc:1034] running on GCE node
23:59:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.612645 21556 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.613504 21556 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:18.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.614727 21556 hybrid_clock.cc:648] HybridClock initialized: now 1783468758614660 us; error 45 us; skew 500 ppm
23:59:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.614989 21556 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:18.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.621008 21556 webserver.cc:460] Webserver started at http://127.19.197.2:33071/ using document root <none> and password file <none>
23:59:18.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.621723 21556 fs_manager.cc:359] Metadata directory not provided
23:59:18.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.621876 21556 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.626894 21556 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.003s
23:59:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.629668 21571 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:18.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.631165 21556 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.631393 21556 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.632473 21556 fs_report.cc:383] FS layout report
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/data
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:18.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.671566 21556 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.672396 21556 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.672730 21556 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:18.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.673700 21556 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:18.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.675670 21578 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.680994 21556 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.681099 21556 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.681197 21556 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.684495 21556 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.684572 21556 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
23:59:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.684960 21578 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap starting.
23:59:18.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.707676 21578 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Log is configured to *not* fsync() on all Append() calls
23:59:18.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.729079 21578 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:18.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.729838 21578 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap complete.
23:59:18.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.731464 21578 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent bootstrapping tablet: real 0.047s	user 0.033s	sys 0.013s
23:59:18.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.742565 21578 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:18.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.744007 21578 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Initialized, Role: FOLLOWER
23:59:18.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.744900 21578 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:18.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.748766 21578 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent starting tablet: real 0.017s	user 0.010s	sys 0.006s
23:59:18.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.752990 21556 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.2:45931
23:59:18.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.753022 21686 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.2:45931 every 8 connection(s)
23:59:18.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.754418 21556 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:18.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.761279 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 21556
23:59:18.761 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:18.773 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:18.774 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.1:36839
23:59:18.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.774674 21689 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.775152 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 21194
23:59:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.775478 21689 heartbeater.cc:464] Registering TS with master...
23:59:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.776399 21689 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.778896 20356 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.781097 21688 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.781234 21688 heartbeater.cc:464] Registering TS with master...
23:59:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.781710 21688 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.782933 20286 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.783746 21687 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.783906 21687 heartbeater.cc:464] Registering TS with master...
23:59:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.784351 21687 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.784386 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.2:52923
23:59:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.785797 20427 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.790524 21318 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:18.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:18.894188 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 21194
23:59:18.912 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] handling channelInactive
23:59:18.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state READY due to: connection closed
23:59:18.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:18.913882 21702 negotiation.cc:337] Failed RPC negotiation. Trace:
23:59:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:18.889531 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.19.197.1:47225 (local address 127.19.197.2:45931)
23:59:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:18.890326 (+   795us) server_negotiation.cc:206] Beginning negotiation
23:59:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:18.890340 (+    14us) server_negotiation.cc:399] Waiting for connection header
23:59:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:18.894166 (+  3826us) server_negotiation.cc:407] Connection header received
23:59:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:18.912534 (+ 18368us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.197.1:47225: BlockingRecv error: recv got EOF from 127.19.197.1:47225 (error 108)
23:59:18.914 [DEBUG - Time-limited test] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":566,"thread_start_us":319,"threads_started":1}
23:59:18.916 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.916 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.916 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.935 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:18.935 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:18.935 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [3ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [3ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@427287835(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@c683720 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1889455f -> wakeup thread Time-limited test)) after lookup
23:59:18.936 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:18.937 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.937 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.937 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.954 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:18.955 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:18.955 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [3ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [3ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [23ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [24ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [24ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [41ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@427287835(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@c683720 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1889455f -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699))) after lookup
23:59:18.956 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:18.957 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.957 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.958 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.975 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:18.975 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:18.976 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [3ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [3ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [23ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [24ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [24ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [41ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [45ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [62ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@427287835(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@c683720 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1889455f -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941))) after lookup
23:59:18.977 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:18.977 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.977 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:18.978 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:19.075 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:19.076 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:19.076 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=163), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [3ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [3ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [23ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [24ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [24ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [41ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [45ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [62ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [63ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [64ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [65ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [160ms] refreshing cache from master, [160ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [162ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@427287835(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@c683720 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941) -> (continuation of Deferred@1742182039 after retry RPC@1604571942), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1889455f -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941) -> (continuation of Deferred@1742182039 after retry RPC@1604571942))) after lookup
23:59:19.077 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:19.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:19.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:19.079 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:19.315 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:19.316 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:19.316 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=402), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [3ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [3ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [23ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [24ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [24ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [41ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [45ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [62ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [63ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [64ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [65ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [160ms] refreshing cache from master, [160ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [162ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [164ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [166ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [167ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [399ms] refreshing cache from master, [399ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [402ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@427287835(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@c683720 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941) -> (continuation of Deferred@1742182039 after retry RPC@1604571942) -> (continuation of Deferred@17763943 after retry RPC@1089787779), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1889455f -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941) -> (continuation of Deferred@1742182039 after retry RPC@1604571942) -> (continuation of Deferred@17763943 after retry RPC@1089787779))) after lookup
23:59:19.317 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:19.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:19.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:19.319 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:19.788817 21688 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.039544 21703 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:59:20.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.039855 21703 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.041529 21703 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:20.043270 21574 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:20.044876 21574 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.048448 21466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.049261 21574 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf
23:59:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.049577 21703 raft_consensus.cc:2726] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
23:59:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.150628 21708 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:20.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.150851 21708 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:20.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.152525 21708 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:20.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:20.154423 21399 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:20.156126 21399 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.161312 21641 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" is_pre_election: true
23:59:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.161690 21641 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 67bf7defb0654350846098210079d01d in term 4.
23:59:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.162622 21399 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.162955 21708 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 5 FOLLOWER]: Leader pre-election won for term 6
23:59:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.163089 21708 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 5 FOLLOWER]: Starting leader election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.163195 21708 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 5 FOLLOWER]: Advancing to term 6
23:59:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.165150 21708 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.165752 21708 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 6 election: Requested vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.166471 21641 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.166723 21641 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 4 FOLLOWER]: Advancing to term 6
23:59:20.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:20.167109 21399 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.170517 21641 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 67bf7defb0654350846098210079d01d in term 6.
23:59:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.170897 21399 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.171192 21708 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 6 FOLLOWER]: Leader election won for term 6
23:59:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.171594 21708 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 6 LEADER]: Becoming Leader. State: Replica: 67bf7defb0654350846098210079d01d, State: Running, Role: LEADER
23:59:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.172065 21708 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:20.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.176260 20286 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d reported cstate change: term changed from 5 to 6, leader changed from 6a4687af738441b284ae2313d7276cbf (127.19.197.1) to 67bf7defb0654350846098210079d01d (127.19.197.3). New cstate: current_term: 6 leader_uuid: "67bf7defb0654350846098210079d01d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: UNKNOWN } } }
23:59:20.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.617287 21641 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 6 FOLLOWER]: Refusing update from remote peer 67bf7defb0654350846098210079d01d: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
23:59:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.618247 21708 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [LEADER]: Connected to new peer: Peer: permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
23:59:20.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:20.632135 21399 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d -> Peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Couldn't send request to peer 6a4687af738441b284ae2313d7276cbf. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:20.816 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:20.816 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:20.817 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1903), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [3ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [3ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [23ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [24ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [24ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [41ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [42ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [45ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [62ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [63ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [64ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [65ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [160ms] refreshing cache from master, [160ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [162ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [164ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [166ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [167ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [399ms] refreshing cache from master, [399ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [402ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [404ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [406ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [406ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [1899ms] refreshing cache from master, [1900ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1903ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@427287835(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@c683720 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941) -> (continuation of Deferred@1742182039 after retry RPC@1604571942) -> (continuation of Deferred@17763943 after retry RPC@1089787779) -> (continuation of Deferred@890644876 after retry RPC@1459040934), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1889455f -> wakeup thread Time-limited test -> (continuation of Deferred@492739146 after retry RPC@947477699) -> (continuation of Deferred@197617802 after retry RPC@2041229941) -> (continuation of Deferred@1742182039 after retry RPC@1604571942) -> (continuation of Deferred@17763943 after retry RPC@1089787779) -> (continuation of Deferred@890644876 after retry RPC@1459040934))) after lookup
23:59:20.818 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:20.819 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] Successfully connected to peer
23:59:20.840 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x4021b033, L:/127.0.0.1:57034 - R:/127.19.197.3:33657] using TOKEN/null
23:59:20.863 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.1:36839
23:59:20.864 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:20.865103 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.188400 21736 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.188730 21736 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.188843 21736 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.196373 21736 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.196589 21736 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.1
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.205751 21736 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.207508 21736 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.209300 21736 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:21.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.222592 21742 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.222631 21744 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.222672 21741 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.223054 21736 server_base.cc:1034] running on GCE node
23:59:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.223909 21736 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.224858 21736 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.226114 21736 hybrid_clock.cc:648] HybridClock initialized: now 1783468761225989 us; error 56 us; skew 500 ppm
23:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.226382 21736 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.228592 21736 webserver.cc:460] Webserver started at http://127.19.197.1:35163/ using document root <none> and password file <none>
23:59:21.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.229192 21736 fs_manager.cc:359] Metadata directory not provided
23:59:21.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.229305 21736 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:21.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.234421 21736 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.001s
23:59:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.237166 21750 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:21.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.238584 21736 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.238750 21736 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.239662 21736 fs_report.cc:383] FS layout report
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/data
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:21.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.268656 21736 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.269554 21736 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.269829 21736 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.270783 21736 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.272740 21757 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:21.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.278239 21736 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:21.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.278370 21736 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:21.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.278462 21736 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.281441 21736 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.281517 21736 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
23:59:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.281929 21757 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap starting.
23:59:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.308619 21757 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Log is configured to *not* fsync() on all Append() calls
23:59:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.326279 21736 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.1:36839
23:59:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.326303 21864 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.1:36839 every 8 connection(s)
23:59:21.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.327589 21736 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.332129 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 21736
23:59:21.333 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:21.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.350061 21757 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:21.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.357656 21757 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap complete.
23:59:21.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.360692 21757 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent bootstrapping tablet: real 0.079s	user 0.052s	sys 0.004s
23:59:21.362 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:21.363 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.3:33657
23:59:21.363 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:59:21.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.364095 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 21382
23:59:21.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.378911 21865 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:21.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.379273 21865 heartbeater.cc:464] Registering TS with master...
23:59:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.380252 21865 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:21.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.383131 20427 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.385627 21867 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.385785 21867 heartbeater.cc:464] Registering TS with master...
23:59:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.386317 21867 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:21.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.387881 20356 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:21.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.389400 21866 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:21.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.389523 21866 heartbeater.cc:464] Registering TS with master...
23:59:21.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.389963 21866 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:21.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.388689 21757 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:21.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.390894 21757 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Initialized, Role: FOLLOWER
23:59:21.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.391672 21757 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:21.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.393925 20286 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:21.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.396037 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.1:43129
23:59:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.399200 21866 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.400776 21757 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent starting tablet: real 0.038s	user 0.012s	sys 0.004s
23:59:21.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.406235 21506 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:21.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:21.506882 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 21382
23:59:21.522 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] handling channelInactive
23:59:21.522 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state READY due to: connection closed
23:59:21.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:21.523118 21881 negotiation.cc:337] Failed RPC negotiation. Trace:
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.505728 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.19.197.3:47717 (local address 127.19.197.1:36839)
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.506481 (+   753us) server_negotiation.cc:206] Beginning negotiation
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.506489 (+     8us) server_negotiation.cc:399] Waiting for connection header
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.506527 (+    38us) server_negotiation.cc:407] Connection header received
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.506649 (+   122us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.506661 (+    12us) server_negotiation.cc:461] Received NEGOTIATE request from client
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.507594 (+   933us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.507791 (+   197us) server_negotiation.cc:226] Negotiated authn=CERTIFICATE
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:21.521543 (+ 13752us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.197.3:47717: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:59:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":579,"thread_start_us":411,"threads_started":1}
23:59:21.523 [DEBUG - Time-limited test] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:21.524 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.524 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.525 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.534 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:21.535 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:21.535 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1288598304(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5d0b0b4d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38be6e51 -> wakeup thread Time-limited test)) after lookup
23:59:21.536 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:21.537 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.537 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.537 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.555 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:21.557 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:21.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server 67bf7defb0654350846098210079d01d, [14ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [15ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1288598304(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5d0b0b4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38be6e51 -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435))) after lookup
23:59:21.557 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:21.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.559 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.575 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:21.576 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:21.576 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server 67bf7defb0654350846098210079d01d, [14ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [15ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [34ms] sending RPC to server 67bf7defb0654350846098210079d01d, [36ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [36ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [52ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1288598304(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5d0b0b4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38be6e51 -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167))) after lookup
23:59:21.577 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:21.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.578 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.655 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:21.655 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:21.656 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=132), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server 67bf7defb0654350846098210079d01d, [14ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [15ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [34ms] sending RPC to server 67bf7defb0654350846098210079d01d, [36ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [36ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [52ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [53ms] sending RPC to server 67bf7defb0654350846098210079d01d, [55ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [56ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [131ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1288598304(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5d0b0b4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167) -> (continuation of Deferred@1111958317 after retry RPC@1172339072), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38be6e51 -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167) -> (continuation of Deferred@1111958317 after retry RPC@1172339072))) after lookup
23:59:21.656 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:21.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.658 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.855 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:21.856 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:21.877 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=333), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server 67bf7defb0654350846098210079d01d, [14ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [15ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [34ms] sending RPC to server 67bf7defb0654350846098210079d01d, [36ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [36ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [52ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [53ms] sending RPC to server 67bf7defb0654350846098210079d01d, [55ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [56ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [131ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [133ms] sending RPC to server 67bf7defb0654350846098210079d01d, [135ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [135ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [329ms] refreshing cache from master, [329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [332ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1288598304(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5d0b0b4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167) -> (continuation of Deferred@1111958317 after retry RPC@1172339072) -> (continuation of Deferred@1232333410 after retry RPC@2140562197), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38be6e51 -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167) -> (continuation of Deferred@1111958317 after retry RPC@1172339072) -> (continuation of Deferred@1232333410 after retry RPC@2140562197))) after lookup
23:59:21.879 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:21.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state CONNECTING due to: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:21.881 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 67bf7defb0654350846098210079d01d(127.19.197.3:33657) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657
23:59:22.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:22.709931 21882 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:59:22.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:22.710268 21882 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:22.711730 21882 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:22.713555 21753 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:22.715906 21753 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:22.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:22.719069 21641 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" is_pre_election: true
23:59:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:22.719825 21753 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a4687af738441b284ae2313d7276cbf; no voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:22.720248 21882 raft_consensus.cc:2726] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
23:59:23.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.006157 21887 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 67bf7defb0654350846098210079d01d)
23:59:23.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.006340 21887 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.006917 21887 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.009927 21574 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:23.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.014876 21819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf" is_pre_election: true
23:59:23.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.015241 21819 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 5.
23:59:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.016227 21574 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 67bf7defb0654350846098210079d01d
23:59:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.016505 21887 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 6 FOLLOWER]: Leader pre-election won for term 7
23:59:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.016618 21887 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 67bf7defb0654350846098210079d01d)
23:59:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.016701 21887 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 6 FOLLOWER]: Advancing to term 7
23:59:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.018947 21887 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.019618 21887 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 7 election: Requested vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.020103 21819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.020329 21819 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 5 FOLLOWER]: Advancing to term 7
23:59:23.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.021149 21574 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111)
23:59:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.024214 21819 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 7.
23:59:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.024569 21574 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 67bf7defb0654350846098210079d01d
23:59:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.024827 21887 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 7 FOLLOWER]: Leader election won for term 7
23:59:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.025252 21887 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 7 LEADER]: Becoming Leader. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Running, Role: LEADER
23:59:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.025722 21887 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.029305 20286 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 reported cstate change: term changed from 6 to 7, leader changed from 67bf7defb0654350846098210079d01d (127.19.197.3) to df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2). New cstate: current_term: 7 leader_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: HEALTHY } } }
23:59:23.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.502522 21574 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 -> Peer 67bf7defb0654350846098210079d01d (127.19.197.3:33657): Couldn't send request to peer 67bf7defb0654350846098210079d01d. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.3:33657: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:23.554 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:23.555 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:23.555 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2032), Traces: [0ms] sending RPC to server 67bf7defb0654350846098210079d01d, [2ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server 67bf7defb0654350846098210079d01d, [14ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [15ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [34ms] sending RPC to server 67bf7defb0654350846098210079d01d, [36ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [36ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [52ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [53ms] sending RPC to server 67bf7defb0654350846098210079d01d, [55ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [56ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [131ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [133ms] sending RPC to server 67bf7defb0654350846098210079d01d, [135ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [135ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [329ms] refreshing cache from master, [329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [332ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [356ms] sending RPC to server 67bf7defb0654350846098210079d01d, [358ms] received response from server 67bf7defb0654350846098210079d01d: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [358ms] delaying RPC due to: Network error: Failed to connect to peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657): Connection refused: /127.19.197.3:33657, [2028ms] refreshing cache from master, [2028ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [2031ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1288598304(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5d0b0b4d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167) -> (continuation of Deferred@1111958317 after retry RPC@1172339072) -> (continuation of Deferred@1232333410 after retry RPC@2140562197) -> (continuation of Deferred@1023459194 after retry RPC@1908600068), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38be6e51 -> wakeup thread Time-limited test -> (continuation of Deferred@1669068395 after retry RPC@679264435) -> (continuation of Deferred@2063454204 after retry RPC@561302167) -> (continuation of Deferred@1111958317 after retry RPC@1172339072) -> (continuation of Deferred@1232333410 after retry RPC@2140562197) -> (continuation of Deferred@1023459194 after retry RPC@1908600068))) after lookup
23:59:23.556 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:23.557 [DEBUG - kudu-nio-0] (Connection.java:757) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] Successfully connected to peer
23:59:23.575 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x58df44de, L:/127.0.0.1:50704 - R:/127.19.197.2:45931] using TOKEN/null
23:59:23.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.579769 21819 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 7 FOLLOWER]: Refusing update from remote peer df2cc700bf114fb3af3f4fb7fdd3b1b0: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
23:59:23.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.580601 21907 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
23:59:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.591224 21906 mvcc.cc:205] Tried to move back new op lower bound from 7305088055615700992 to 7305088053357412352. Current Snapshot: MvccSnapshot[applied={T|T < 7305088055615700992}]
23:59:23.612 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.3:33657
23:59:23.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.613205 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/logs
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.3:33657
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.3
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.3
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.943331 21917 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.943709 21917 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.943835 21917 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:23.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.951349 21917 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:23.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.951507 21917 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.3
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.960826 21917 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.3:33657
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.3
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.3
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/logs
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.962108 21917 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:23.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.963877 21917 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.976837 21922 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.976984 21923 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:23.976809 21925 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.977538 21917 server_base.cc:1034] running on GCE node
23:59:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.978176 21917 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.979086 21917 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.980336 21917 hybrid_clock.cc:648] HybridClock initialized: now 1783468763980264 us; error 34 us; skew 500 ppm
23:59:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.980649 21917 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:23.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.987178 21917 webserver.cc:460] Webserver started at http://127.19.197.3:41827/ using document root <none> and password file <none>
23:59:23.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.987870 21917 fs_manager.cc:359] Metadata directory not provided
23:59:23.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.988013 21917 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:23.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.993038 21917 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.005s	sys 0.000s
23:59:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.996009 21931 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.997408 21917 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
23:59:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.997598 21917 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67bf7defb0654350846098210079d01d"
23:59:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:05 on dist-test-slave-x4hg"
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:23.998821 21917 fs_report.cc:383] FS layout report
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/wal
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/data
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.033247 21917 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:24.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.034231 21917 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:24.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.034538 21917 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.035516 21917 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:24.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.037870 21940 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.043565 21917 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.043718 21917 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.043840 21917 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.047024 21917 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.047122 21917 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
23:59:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.047484 21940 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Bootstrap starting.
23:59:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.073849 21940 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Log is configured to *not* fsync() on all Append() calls
23:59:24.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.095669 21917 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.3:33657
23:59:24.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.095692 22047 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.3:33657 every 8 connection(s)
23:59:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.096982 21917 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-2/data/info.pb
23:59:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.100952 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 21917
23:59:24.102 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:24.112 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:97)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
23:59:24.113 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:79)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
23:59:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.123598 21940 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:24.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.124243 21940 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Bootstrap complete.
23:59:24.127 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:24.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.125700 21940 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Time spent bootstrapping tablet: real 0.079s	user 0.048s	sys 0.009s
23:59:24.127 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.2:45931
23:59:24.127 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:24.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.128073 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 21556
23:59:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.141526 22048 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.141880 22048 heartbeater.cc:464] Registering TS with master...
23:59:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.143112 22048 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:24.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.146476 22049 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:24.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.146605 22049 heartbeater.cc:464] Registering TS with master...
23:59:24.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.146621 20428 ts_manager.cc:194] Re-registered known tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:24.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.147150 22049 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.148703 20286 ts_manager.cc:194] Re-registered known tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.150054 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.3:45377
23:59:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.151808 22050 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.151958 22050 heartbeater.cc:464] Registering TS with master...
23:59:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.153456 22050 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:24.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.153338 21940 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.154614 21940 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 67bf7defb0654350846098210079d01d, State: Initialized, Role: FOLLOWER
23:59:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.155273 21940 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:24.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.160256 20356 ts_manager.cc:194] Re-registered known tserver with Master: 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.167330 22049 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.167583 21940 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d: Time spent starting tablet: real 0.042s	user 0.015s	sys 0.010s
23:59:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.172807 21682 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:24.282852 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 21556
23:59:24.302 [DEBUG - kudu-nio-0] (Connection.java:239) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] handling channelInactive
23:59:24.302 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state READY due to: connection closed
23:59:24.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:24.304 [DEBUG - Time-limited test] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:24.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.305 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.314 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:24.315 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:24.315 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test)) after lookup
23:59:24.316 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:24.317 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.317 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.317 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.335 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:24.335 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:24.336 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [14ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [14ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233))) after lookup
23:59:24.337 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:24.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.338 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.375 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:24.375 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:24.375 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [14ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [14ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [35ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [35ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [71ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175))) after lookup
23:59:24.376 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:24.376 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.377 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.455 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:24.455 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:24.455 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=152), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [14ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [14ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [35ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [35ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [71ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [72ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [74ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [74ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [151ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477))) after lookup
23:59:24.456 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:24.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.458 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.755 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:24.756 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:24.756 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=452), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [14ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [14ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [35ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [35ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [71ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [72ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [74ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [74ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [151ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [153ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [155ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [155ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [449ms] refreshing cache from master, [449ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [452ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300))) after lookup
23:59:24.757 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:24.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.759 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.975 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:24.975 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:24.976 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=672), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [14ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [14ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [35ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [35ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [71ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [72ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [74ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [74ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [151ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [153ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [155ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [155ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [449ms] refreshing cache from master, [449ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [452ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [454ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [455ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [456ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [672ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300) -> (continuation of Deferred@1531511044 after retry RPC@233757910), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300) -> (continuation of Deferred@1531511044 after retry RPC@233757910))) after lookup
23:59:24.977 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:24.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:24.978 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:25.435 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:25.435 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:25.436 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1132), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [14ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [14ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [35ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [35ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [71ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [72ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [74ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [74ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [151ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [153ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [155ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [155ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [449ms] refreshing cache from master, [449ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [452ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [454ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [455ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [456ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [672ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [673ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [675ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [675ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1129ms] refreshing cache from master, [1129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1132ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300) -> (continuation of Deferred@1531511044 after retry RPC@233757910) -> (continuation of Deferred@981389394 after retry RPC@1091416999), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300) -> (continuation of Deferred@1531511044 after retry RPC@233757910) -> (continuation of Deferred@981389394 after retry RPC@1091416999))) after lookup
23:59:25.437 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:25.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:25.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:25.438 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:25.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.561178 22063 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 7 FOLLOWER]: Starting pre-election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:25.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.561416 22063 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:25.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.562182 22063 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:25.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:25.565011 21753 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.570393 22002 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.570889 22002 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 6.
23:59:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.571897 21753 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.572196 22063 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 7 FOLLOWER]: Leader pre-election won for term 8
23:59:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.572305 22063 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 7 FOLLOWER]: Starting leader election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.572366 22063 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 7 FOLLOWER]: Advancing to term 8
23:59:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.574297 22063 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.574749 22063 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 8 election: Requested vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:25.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.575242 22002 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d"
23:59:25.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.575462 22002 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 6 FOLLOWER]: Advancing to term 8
23:59:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:25.576254 21753 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:25.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.579376 22002 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 8.
23:59:25.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.579788 21753 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:25.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.580065 22063 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 8 FOLLOWER]: Leader election won for term 8
23:59:25.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.580510 22063 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 8 LEADER]: Becoming Leader. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Running, Role: LEADER
23:59:25.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.581038 22063 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:25.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:25.584743 20286 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf reported cstate change: term changed from 7 to 8, leader changed from df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2) to 6a4687af738441b284ae2313d7276cbf (127.19.197.1). New cstate: current_term: 8 leader_uuid: "6a4687af738441b284ae2313d7276cbf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: UNKNOWN } } }
23:59:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:26.045151 21753 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf -> Peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Couldn't send request to peer df2cc700bf114fb3af3f4fb7fdd3b1b0. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:26.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:26.156243 22002 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 8 FOLLOWER]: Refusing update from remote peer 6a4687af738441b284ae2313d7276cbf: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
23:59:26.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:26.157148 22063 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Connected to new peer: Peer: permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
23:59:27.374 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:27.375 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:27.376 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=16, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3072), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [2ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [11ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [12ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [14ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [14ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [35ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [35ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [71ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [72ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [74ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [74ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [151ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [153ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [155ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [155ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [449ms] refreshing cache from master, [449ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [452ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [454ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [455ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [456ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [672ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [673ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [675ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [675ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1129ms] refreshing cache from master, [1129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1132ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [1133ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [1135ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1135ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3069ms] refreshing cache from master, [3069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [3071ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1253259113(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@473cefbe -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300) -> (continuation of Deferred@1531511044 after retry RPC@233757910) -> (continuation of Deferred@981389394 after retry RPC@1091416999) -> (continuation of Deferred@18530678 after retry RPC@1219903999), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@226f2b53 -> wakeup thread Time-limited test -> (continuation of Deferred@222237298 after retry RPC@1984812233) -> (continuation of Deferred@751279482 after retry RPC@884277175) -> (continuation of Deferred@852291133 after retry RPC@137901477) -> (continuation of Deferred@515169802 after retry RPC@1466451300) -> (continuation of Deferred@1531511044 after retry RPC@233757910) -> (continuation of Deferred@981389394 after retry RPC@1091416999) -> (continuation of Deferred@18530678 after retry RPC@1219903999))) after lookup
23:59:27.376 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:27.378 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] Successfully connected to peer
23:59:27.396 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x372029e2, L:/127.0.0.1:39914 - R:/127.19.197.1:36839] using TOKEN/null
23:59:27.431 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.2:45931
23:59:27.432 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.433013 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:27.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.756048 22100 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:27.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.756352 22100 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:27.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.756464 22100 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.764156 22100 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.764367 22100 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.2
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.774375 22100 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.775966 22100 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.777715 22100 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:27.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.791033 22107 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:27.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.791011 22108 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:27.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:27.791081 22110 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:27.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.791723 22100 server_base.cc:1034] running on GCE node
23:59:27.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.792275 22100 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:27.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.793184 22100 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:27.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.794459 22100 hybrid_clock.cc:648] HybridClock initialized: now 1783468767794364 us; error 58 us; skew 500 ppm
23:59:27.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.794773 22100 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.797026 22100 webserver.cc:460] Webserver started at http://127.19.197.2:33071/ using document root <none> and password file <none>
23:59:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.797676 22100 fs_manager.cc:359] Metadata directory not provided
23:59:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.797781 22100 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.802583 22100 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.002s
23:59:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.805418 22116 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.806707 22100 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:59:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.806898 22100 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.807934 22100 fs_report.cc:383] FS layout report
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/data
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:27.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.828400 22100 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:27.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.829288 22100 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:27.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.829608 22100 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:27.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.830643 22100 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:27.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.832564 22123 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:27.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.837765 22100 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:27.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.837903 22100 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:27.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.838019 22100 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.840917 22100 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.841022 22100 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
23:59:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.841394 22123 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap starting.
23:59:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.868609 22123 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Log is configured to *not* fsync() on all Append() calls
23:59:27.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.905328 22123 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:27.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.906013 22123 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap complete.
23:59:27.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.907552 22123 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent bootstrapping tablet: real 0.066s	user 0.048s	sys 0.016s
23:59:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.919488 22100 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.2:45931
23:59:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.918383 22123 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.919525 22230 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.2:45931 every 8 connection(s)
23:59:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.919690 22123 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Initialized, Role: FOLLOWER
23:59:27.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.920799 22123 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:27.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.921289 22100 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.923550 22123 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent starting tablet: real 0.016s	user 0.013s	sys 0.002s
23:59:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.929864 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 22100
23:59:27.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:27.956 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:27.958 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.1:36839
23:59:27.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:27.958706 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 21736
23:59:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.008538 22232 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.009004 22232 heartbeater.cc:464] Registering TS with master...
23:59:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.010562 22233 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.010739 22233 heartbeater.cc:464] Registering TS with master...
23:59:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.011499 22233 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:28.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.015440 22232 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:28.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.015962 20285 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.017530 20285 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.2:52639
23:59:28.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.019698 20427 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.027343 22237 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.027657 22237 heartbeater.cc:464] Registering TS with master...
23:59:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.028160 22237 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:28.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.029651 20356 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.037477 22186 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 7 FOLLOWER]: Advancing to term 8
23:59:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.042519 22233 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.043161 21860 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.047209 22247 mvcc.cc:205] Tried to move back new op lower bound from 7305088071270928384 to 7305088063823847424. Current Snapshot: MvccSnapshot[applied={T|T < 7305088071270928384}]
23:59:28.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:28.158829 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 21736
23:59:28.181 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] handling channelInactive
23:59:28.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state READY due to: connection closed
23:59:28.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:28.182 [DEBUG - Time-limited test] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:28.184 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.184 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.184 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.194 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:28.194 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:28.195 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [12ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1968413044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@618c99b3 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11f5614c -> wakeup thread Time-limited test)) after lookup
23:59:28.195 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:28.197 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.197 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.214 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:28.214 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:28.215 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [12ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [13ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [15ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1968413044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@618c99b3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11f5614c -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113))) after lookup
23:59:28.215 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:28.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.216 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.255 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:28.255 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:28.255 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [12ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [13ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [15ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [34ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [34ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [72ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1968413044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@618c99b3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11f5614c -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107))) after lookup
23:59:28.256 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:28.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.258 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.355 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:28.355 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:28.355 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=173), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [12ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [13ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [15ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [34ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [34ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [72ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [74ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [76ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [76ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [172ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1968413044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@618c99b3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107) -> (continuation of Deferred@1706668761 after retry RPC@1380635006), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11f5614c -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107) -> (continuation of Deferred@1706668761 after retry RPC@1380635006))) after lookup
23:59:28.356 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:28.357 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.357 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.357 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.455 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:28.455 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:28.455 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=273), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [12ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [13ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [15ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [34ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [34ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [72ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [74ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [76ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [76ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [172ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [174ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [175ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [175ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [272ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1968413044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@618c99b3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107) -> (continuation of Deferred@1706668761 after retry RPC@1380635006) -> (continuation of Deferred@722783779 after retry RPC@904436529), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11f5614c -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107) -> (continuation of Deferred@1706668761 after retry RPC@1380635006) -> (continuation of Deferred@722783779 after retry RPC@904436529))) after lookup
23:59:28.456 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:28.457 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:28.458 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.426318 22251 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.426553 22251 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:29.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.428057 22251 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:29.429803 21934 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:29.432271 21934 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.434928 22186 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" is_pre_election: true
23:59:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.435886 21934 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 67bf7defb0654350846098210079d01d; no voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.436316 22251 raft_consensus.cc:2726] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
23:59:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.543785 22256 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.544014 22256 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.545522 22256 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:29.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:29.546976 22119 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:29.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:29.548792 22119 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:29.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.553434 22002 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:29.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.553639 22002 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 8.
23:59:29.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.554168 22119 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:29.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.554515 22256 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 8 FOLLOWER]: Leader pre-election won for term 9
23:59:29.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.554639 22256 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:29.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.554740 22256 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 8 FOLLOWER]: Advancing to term 9
23:59:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.556680 22256 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.557251 22256 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 9 election: Requested vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), 67bf7defb0654350846098210079d01d (127.19.197.3:33657)
23:59:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.558018 22002 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d"
23:59:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.558287 22002 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 8 FOLLOWER]: Advancing to term 9
23:59:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:29.558578 22119 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.560308 22002 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate df2cc700bf114fb3af3f4fb7fdd3b1b0 in term 9.
23:59:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.560704 22119 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:29.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.560994 22256 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 9 FOLLOWER]: Leader election won for term 9
23:59:29.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.561424 22256 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 9 LEADER]: Becoming Leader. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Running, Role: LEADER
23:59:29.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.561853 22256 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:29.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.565259 20285 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 reported cstate change: term changed from 8 to 9, leader changed from 6a4687af738441b284ae2313d7276cbf (127.19.197.1) to df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2). New cstate: current_term: 9 leader_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: HEALTHY } } }
23:59:29.735 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:29.735 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:29.736 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1553), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [12ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [13ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [15ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [32ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [33ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [34ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [34ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [72ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [74ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [76ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [76ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [172ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [174ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [175ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [175ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [272ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [274ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [276ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [276ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [1550ms] refreshing cache from master, [1550ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1552ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1968413044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@618c99b3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107) -> (continuation of Deferred@1706668761 after retry RPC@1380635006) -> (continuation of Deferred@722783779 after retry RPC@904436529) -> (continuation of Deferred@335220802 after retry RPC@1648042015), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11f5614c -> wakeup thread Time-limited test -> (continuation of Deferred@1867113285 after retry RPC@1426854113) -> (continuation of Deferred@1867704298 after retry RPC@1549624107) -> (continuation of Deferred@1706668761 after retry RPC@1380635006) -> (continuation of Deferred@722783779 after retry RPC@904436529) -> (continuation of Deferred@335220802 after retry RPC@1648042015))) after lookup
23:59:29.737 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:29.737 [DEBUG - kudu-nio-0] (Connection.java:757) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] Successfully connected to peer
23:59:29.753 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0faf32a2, L:/127.0.0.1:41000 - R:/127.19.197.2:45931] using TOKEN/null
23:59:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.757807 22002 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 9 FOLLOWER]: Refusing update from remote peer df2cc700bf114fb3af3f4fb7fdd3b1b0: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 9 index: 34. (index mismatch)
23:59:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.758912 22261 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 32, Time since last communication: 0.000s
23:59:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:29.759241 22119 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 -> Peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Couldn't send request to peer 6a4687af738441b284ae2313d7276cbf. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.767421 22274 mvcc.cc:205] Tried to move back new op lower bound from 7305088080920313856 to 7305088080128540672. Current Snapshot: MvccSnapshot[applied={T|T < 7305088080920313856 or (T in {7305088080920313856})}]
23:59:29.782 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.1:36839
23:59:29.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:29.783421 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:30.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.101370 22279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:30.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.101636 22279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:30.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.101714 22279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.109468 22279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.109714 22279 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.1
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.118793 22279 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.119979 22279 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.121637 22279 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:30.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.134366 22284 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:30.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.134419 22285 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:30.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.134393 22287 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.135221 22279 server_base.cc:1034] running on GCE node
23:59:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.135751 22279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.136626 22279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.137867 22279 hybrid_clock.cc:648] HybridClock initialized: now 1783468770137815 us; error 40 us; skew 500 ppm
23:59:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.138202 22279 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.143743 22279 webserver.cc:460] Webserver started at http://127.19.197.1:35163/ using document root <none> and password file <none>
23:59:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.144433 22279 fs_manager.cc:359] Metadata directory not provided
23:59:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.144575 22279 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.149901 22279 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.002s	sys 0.001s
23:59:30.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.152611 22293 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:30.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.154161 22279 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.154327 22279 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.155571 22279 fs_report.cc:383] FS layout report
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/data
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.175025 22279 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.175881 22279 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:30.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.176110 22279 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:30.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.176972 22279 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.178959 22300 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:30.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.184314 22279 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:30.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.184466 22279 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:30.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.184567 22279 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.187796 22279 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.187866 22279 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.002s	sys 0.000s
23:59:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.188272 22300 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap starting.
23:59:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.222195 22300 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Log is configured to *not* fsync() on all Append() calls
23:59:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.224510 22279 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.1:36839
23:59:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.224532 22407 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.1:36839 every 8 connection(s)
23:59:30.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.226002 22279 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.228447 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 22279
23:59:30.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:30.255 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:30.256 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.2:45931
23:59:30.256 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:30.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.256883 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 22100
23:59:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.304463 22409 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.304773 22409 heartbeater.cc:464] Registering TS with master...
23:59:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.305680 22300 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.305806 22409 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.305986 22410 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.306143 22410 heartbeater.cc:464] Registering TS with master...
23:59:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.306439 22300 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap complete.
23:59:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.306646 22410 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.307828 22408 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.307906 22300 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent bootstrapping tablet: real 0.120s	user 0.062s	sys 0.020s
23:59:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.307967 22408 heartbeater.cc:464] Registering TS with master...
23:59:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.308455 22408 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:30.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.310425 22226 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:30.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.310544 20428 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.314303 20285 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.314998 20356 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.315733 20285 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.1:34833
23:59:30.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.316530 22300 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:30.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.317813 22300 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Initialized, Role: FOLLOWER
23:59:30.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.318468 22300 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.320678 22300 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent starting tablet: real 0.013s	user 0.008s	sys 0.005s
23:59:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.320735 22409 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:30.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.497136 22362 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 8 FOLLOWER]: Advancing to term 9
23:59:30.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:30.498169 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 22100
23:59:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:30.525470 22296 connection.cc:535] server connection from 127.19.197.2:40735 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:59:30.526 [DEBUG - kudu-nio-0] (Connection.java:239) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] handling channelInactive
23:59:30.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state READY due to: connection closed
23:59:30.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:30.527 [DEBUG - Time-limited test] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:30.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.529 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.535 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:30.535 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:30.535 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [7ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@345672245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@45b0a3c5 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e88a839 -> wakeup thread Time-limited test)) after lookup
23:59:30.536 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:30.537 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.537 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.537 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.554 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:30.554 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:30.555 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [7ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [8ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [10ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [10ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [27ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@345672245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@45b0a3c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e88a839 -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946))) after lookup
23:59:30.555 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:30.557 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.557 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.557 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.575 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:30.575 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:30.576 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [7ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [8ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [10ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [10ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [27ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [30ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [30ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [48ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@345672245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@45b0a3c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e88a839 -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158))) after lookup
23:59:30.577 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:30.577 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.577 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.578 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.615 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:30.615 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:30.616 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [7ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [8ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [10ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [10ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [27ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [30ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [30ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [48ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [49ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [51ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [51ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [87ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@345672245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@45b0a3c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e88a839 -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822))) after lookup
23:59:30.617 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:30.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.619 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.975 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:30.975 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:30.976 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=448), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [7ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [8ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [10ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [10ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [27ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [30ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [30ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [48ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [49ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [51ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [51ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [87ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [90ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [92ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [92ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [445ms] refreshing cache from master, [445ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [448ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@345672245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@45b0a3c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822) -> (continuation of Deferred@712207368 after retry RPC@1403769619), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e88a839 -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822) -> (continuation of Deferred@712207368 after retry RPC@1403769619))) after lookup
23:59:30.977 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:30.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:30.978 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:31.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:31.282557 22430 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 9 FOLLOWER]: Starting pre-election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:31.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:31.282773 22430 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:31.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:31.283671 22430 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:31.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:31.286540 21934 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:31.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:31.290287 22362 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "6a4687af738441b284ae2313d7276cbf" is_pre_election: true
23:59:31.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:31.291116 21934 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 67bf7defb0654350846098210079d01d; no voters: 6a4687af738441b284ae2313d7276cbf, df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:31.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:31.291414 22430 raft_consensus.cc:2726] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
23:59:31.595 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:31.595 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)[L]]}]
23:59:31.596 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1068), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [7ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [8ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [10ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [10ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [27ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [30ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [30ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [48ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [49ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [51ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [51ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [87ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [90ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [92ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [92ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [445ms] refreshing cache from master, [445ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [448ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [449ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [451ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [451ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1065ms] refreshing cache from master, [1065ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1067ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@345672245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@45b0a3c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822) -> (continuation of Deferred@712207368 after retry RPC@1403769619) -> (continuation of Deferred@677442609 after retry RPC@134382858), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e88a839 -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822) -> (continuation of Deferred@712207368 after retry RPC@1403769619) -> (continuation of Deferred@677442609 after retry RPC@134382858))) after lookup
23:59:31.596 [DEBUG - kudu-nio-0] (Connection.java:748) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] connecting to peer
23:59:31.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:31.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] cleaning up while in state CONNECTING due to: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:31.598 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931
23:59:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.004118 22435 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 9 FOLLOWER]: Starting pre-election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.004395 22435 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.005730 22435 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:32.007956 22296 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:32.009478 22296 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.012039 22002 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d" is_pre_election: true
23:59:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.012293 22002 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 9.
23:59:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.012830 22296 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.013204 22435 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 9 FOLLOWER]: Leader pre-election won for term 10
23:59:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.013311 22435 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 9 FOLLOWER]: Starting leader election (detected failure of leader df2cc700bf114fb3af3f4fb7fdd3b1b0)
23:59:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.013387 22435 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 9 FOLLOWER]: Advancing to term 10
23:59:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.015321 22435 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:32.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.015849 22435 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 10 election: Requested vote from peers 67bf7defb0654350846098210079d01d (127.19.197.3:33657), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:32.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.016295 22002 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "6a4687af738441b284ae2313d7276cbf" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "67bf7defb0654350846098210079d01d"
23:59:32.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.016479 22002 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 9 FOLLOWER]: Advancing to term 10
23:59:32.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:32.017306 22296 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111)
23:59:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.018440 22002 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a4687af738441b284ae2313d7276cbf in term 10.
23:59:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.018813 22296 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, 6a4687af738441b284ae2313d7276cbf; no voters: df2cc700bf114fb3af3f4fb7fdd3b1b0
23:59:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.019078 22435 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 10 FOLLOWER]: Leader election won for term 10
23:59:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.019500 22435 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [term 10 LEADER]: Becoming Leader. State: Replica: 6a4687af738441b284ae2313d7276cbf, State: Running, Role: LEADER
23:59:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.019992 22435 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:32.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.023347 20285 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf reported cstate change: term changed from 9 to 10, leader changed from df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2) to 6a4687af738441b284ae2313d7276cbf (127.19.197.1). New cstate: current_term: 10 leader_uuid: "6a4687af738441b284ae2313d7276cbf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: UNKNOWN } } }
23:59:32.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.526139 22002 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 10 FOLLOWER]: Refusing update from remote peer 6a4687af738441b284ae2313d7276cbf: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
23:59:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:32.526868 22452 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf [LEADER]: Connected to new peer: Peer: permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 36, Time since last communication: 0.000s
23:59:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:32.537197 22296 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf -> Peer df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931): Couldn't send request to peer df2cc700bf114fb3af3f4fb7fdd3b1b0. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.2:45931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:33.795 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:33.796 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:33.796 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=3269), Traces: [0ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [2ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [7ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [8ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [10ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [10ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [27ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [28ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [30ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [30ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [48ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [49ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [51ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [51ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [87ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [90ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [92ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [92ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [445ms] refreshing cache from master, [445ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [448ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [449ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [451ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [451ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1065ms] refreshing cache from master, [1065ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [1067ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [1069ms] sending RPC to server df2cc700bf114fb3af3f4fb7fdd3b1b0, [1071ms] received response from server df2cc700bf114fb3af3f4fb7fdd3b1b0: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [1071ms] delaying RPC due to: Network error: Failed to connect to peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931): Connection refused: /127.19.197.2:45931, [3265ms] refreshing cache from master, [3265ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [3268ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@345672245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@45b0a3c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822) -> (continuation of Deferred@712207368 after retry RPC@1403769619) -> (continuation of Deferred@677442609 after retry RPC@134382858) -> (continuation of Deferred@1498334964 after retry RPC@1165505026), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4e88a839 -> wakeup thread Time-limited test -> (continuation of Deferred@1500604254 after retry RPC@1763495946) -> (continuation of Deferred@636383737 after retry RPC@712862158) -> (continuation of Deferred@1891699232 after retry RPC@50297822) -> (continuation of Deferred@712207368 after retry RPC@1403769619) -> (continuation of Deferred@677442609 after retry RPC@134382858) -> (continuation of Deferred@1498334964 after retry RPC@1165505026))) after lookup
23:59:33.797 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:33.798 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] Successfully connected to peer
23:59:33.812 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb9df508b, L:/127.0.0.1:43318 - R:/127.19.197.1:36839] using TOKEN/null
23:59:33.833 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.2:45931
23:59:33.833 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:33.834337 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.160782 22471 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.161082 22471 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.161162 22471 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:34.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.168248 22471 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:34.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.168411 22471 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.2
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.177315 22471 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.2:45931
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.2
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33071
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.2
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/logs
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.178766 22471 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.180620 22471 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:34.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.193753 22478 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:34.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.193965 22479 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:34.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.194517 22481 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:34.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.194911 22471 server_base.cc:1034] running on GCE node
23:59:34.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.195487 22471 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.196477 22471 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.197717 22471 hybrid_clock.cc:648] HybridClock initialized: now 1783468774197610 us; error 76 us; skew 500 ppm
23:59:34.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.198092 22471 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:34.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.200238 22471 webserver.cc:460] Webserver started at http://127.19.197.2:33071/ using document root <none> and password file <none>
23:59:34.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.200896 22471 fs_manager.cc:359] Metadata directory not provided
23:59:34.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.201035 22471 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.206005 22471 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.003s	sys 0.000s
23:59:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.208787 22487 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:34.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.210023 22471 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.210243 22471 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.211180 22471 fs_report.cc:383] FS layout report
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/wal
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/data
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.234036 22471 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.234936 22471 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.235245 22471 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.236209 22471 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:34.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.238210 22494 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.243469 22471 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.243594 22471 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.243714 22471 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.246950 22471 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.247053 22471 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
23:59:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.247455 22494 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap starting.
23:59:34.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.275825 22494 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Log is configured to *not* fsync() on all Append() calls
23:59:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.290448 22471 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.2:45931
23:59:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.290470 22601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.2:45931 every 8 connection(s)
23:59:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.291631 22471 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-1/data/info.pb
23:59:34.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.299182 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 22471
23:59:34.299 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:34.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.325139 22494 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:34.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.325846 22494 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Bootstrap complete.
23:59:34.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.327409 22494 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent bootstrapping tablet: real 0.080s	user 0.069s	sys 0.009s
23:59:34.331 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:34.331 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.19.197.1:36839
23:59:34.331 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:34.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.332158 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 22279
23:59:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.339680 22494 raft_consensus.cc:348] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.341040 22494 raft_consensus.cc:729] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: df2cc700bf114fb3af3f4fb7fdd3b1b0, State: Initialized, Role: FOLLOWER
23:59:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.341885 22494 consensus_queue.cc:260] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:34.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.357064 22608 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:34.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.357501 22608 heartbeater.cc:464] Registering TS with master...
23:59:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.358621 22608 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:34.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.362089 20357 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:34.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.362593 22494 ts_tablet_manager.cc:1428] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0: Time spent starting tablet: real 0.035s	user 0.021s	sys 0.003s
23:59:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.365741 22402 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.369463 22603 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.369596 22603 heartbeater.cc:464] Registering TS with master...
23:59:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.370070 22603 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.370550 22602 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.370683 22602 heartbeater.cc:464] Registering TS with master...
23:59:34.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.371047 22602 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:34.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.371395 20285 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:34.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.372298 20428 ts_manager.cc:194] Re-registered known tserver with Master: df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.372855 20285 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.2:49673
23:59:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.375699 22603 heartbeater.cc:503] Master 127.19.197.62:35395 was elected leader, sending a full tablet report...
23:59:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:34.465986 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 22279
23:59:34.484 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] handling channelInactive
23:59:34.484 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state READY due to: connection closed
23:59:34.485 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:34.485674 22616 negotiation.cc:337] Failed RPC negotiation. Trace:
23:59:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.354794 (+     0us) reactor.cc:625] Submitting negotiation task for server connection from 127.19.197.1:37267 (local address 127.19.197.2:45931)
23:59:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.362529 (+  7735us) server_negotiation.cc:206] Beginning negotiation
23:59:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.362539 (+    10us) server_negotiation.cc:399] Waiting for connection header
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.362574 (+    35us) server_negotiation.cc:407] Connection header received
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.362686 (+   112us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
23:59:34.486 [DEBUG - Time-limited test] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.362695 (+     9us) server_negotiation.cc:461] Received NEGOTIATE request from client
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.362961 (+   266us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.363145 (+   184us) server_negotiation.cc:226] Negotiated authn=SASL
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.363656 (+   511us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.365276 (+  1620us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:34.484155 (+118879us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.197.1:37267: BlockingRecv error: recv got EOF from 127.19.197.1:37267 (error 108)
23:59:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"mutex_wait_us":1,"server-negotiator.queue_time_us":7569,"thread_start_us":14334,"threads_started":1}
23:59:34.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.487 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.487 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.494 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:34.494 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:34.495 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [9ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1183255261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5149ffdc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@241b1c09 -> wakeup thread Time-limited test)) after lookup
23:59:34.495 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:34.496 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.496 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.496 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.515 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:34.515 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:34.515 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [9ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [10ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [11ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [29ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1183255261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5149ffdc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@241b1c09 -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660))) after lookup
23:59:34.516 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:34.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.517 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.517 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.554 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:34.554 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:34.555 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [9ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [10ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [11ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [29ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [30ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [32ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [32ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [69ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1183255261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5149ffdc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@241b1c09 -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553))) after lookup
23:59:34.555 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:34.556 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.556 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.556 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.675 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:34.675 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:34.675 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=190), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [9ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [10ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [11ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [29ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [30ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [32ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [32ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [69ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [70ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [71ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [71ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [189ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1183255261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5149ffdc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@241b1c09 -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580))) after lookup
23:59:34.676 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:34.677 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.677 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.677 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.874 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:34.875 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:34.875 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=390), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [9ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [10ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [11ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [29ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [30ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [32ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [32ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [69ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [70ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [71ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [71ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [189ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [190ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [192ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [192ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [386ms] refreshing cache from master, [387ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [389ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1183255261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5149ffdc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580) -> (continuation of Deferred@1735342974 after retry RPC@962477617), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@241b1c09 -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580) -> (continuation of Deferred@1735342974 after retry RPC@962477617))) after lookup
23:59:34.876 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:34.877 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.877 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.877 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.954 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:34.955 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657),6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)[L],df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:34.955 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=469), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [9ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [10ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [11ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [29ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [30ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [32ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [32ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [69ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [70ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [71ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [71ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [189ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [190ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [192ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [192ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [386ms] refreshing cache from master, [387ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [389ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [391ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [392ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [392ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [466ms] refreshing cache from master, [467ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [469ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1183255261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5149ffdc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580) -> (continuation of Deferred@1735342974 after retry RPC@962477617) -> (continuation of Deferred@1685408287 after retry RPC@474979577), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@241b1c09 -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580) -> (continuation of Deferred@1735342974 after retry RPC@962477617) -> (continuation of Deferred@1685408287 after retry RPC@474979577))) after lookup
23:59:34.956 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] connecting to peer
23:59:34.957 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.957 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:34.957 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839) for tablet 444a22f6bc684186b6cc99f4d034b72b: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839
23:59:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.817487 22618 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.817677 22618 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.818470 22618 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:35.820120 21934 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111) [suppressed 1 similar messages]
23:59:35.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:35.821900 21934 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:35.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.827976 22556 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" is_pre_election: true
23:59:35.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.828470 22556 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 67bf7defb0654350846098210079d01d in term 9.
23:59:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.829465 21934 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.829805 22618 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 10 FOLLOWER]: Leader pre-election won for term 11
23:59:35.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.829948 22618 raft_consensus.cc:482] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 10 FOLLOWER]: Starting leader election (detected failure of leader 6a4687af738441b284ae2313d7276cbf)
23:59:35.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.830040 22618 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 10 FOLLOWER]: Advancing to term 11
23:59:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.833017 22618 raft_consensus.cc:504] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.833673 22618 leader_election.cc:288] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 11 election: Requested vote from peers 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839), df2cc700bf114fb3af3f4fb7fdd3b1b0 (127.19.197.2:45931)
23:59:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.834640 22556 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "444a22f6bc684186b6cc99f4d034b72b" candidate_uuid: "67bf7defb0654350846098210079d01d" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0"
23:59:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.834851 22556 raft_consensus.cc:3037] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 9 FOLLOWER]: Advancing to term 11
23:59:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:35.835281 21934 leader_election.cc:334] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111)
23:59:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.840240 22556 raft_consensus.cc:2454] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 67bf7defb0654350846098210079d01d in term 11.
23:59:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.840772 21934 leader_election.cc:302] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 67bf7defb0654350846098210079d01d, df2cc700bf114fb3af3f4fb7fdd3b1b0; no voters: 6a4687af738441b284ae2313d7276cbf
23:59:35.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.841092 22618 raft_consensus.cc:2781] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 11 FOLLOWER]: Leader election won for term 11
23:59:35.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.841641 22618 raft_consensus.cc:686] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [term 11 LEADER]: Becoming Leader. State: Replica: 67bf7defb0654350846098210079d01d, State: Running, Role: LEADER
23:59:35.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.842306 22618 consensus_queue.cc:237] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } }
23:59:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:35.845849 20285 catalog_manager.cc:5495] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d reported cstate change: term changed from 10 to 11, leader changed from 6a4687af738441b284ae2313d7276cbf (127.19.197.1) to 67bf7defb0654350846098210079d01d (127.19.197.3). New cstate: current_term: 11 leader_uuid: "67bf7defb0654350846098210079d01d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a4687af738441b284ae2313d7276cbf" member_type: VOTER last_known_addr { host: "127.19.197.1" port: 36839 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "67bf7defb0654350846098210079d01d" member_type: VOTER last_known_addr { host: "127.19.197.3" port: 33657 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 } health_report { overall_health: UNKNOWN } } }
23:59:36.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:36.236373 21934 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d -> Peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Couldn't send request to peer 6a4687af738441b284ae2313d7276cbf. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:59:36.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:36.459592 22556 raft_consensus.cc:1261] T 444a22f6bc684186b6cc99f4d034b72b P df2cc700bf114fb3af3f4fb7fdd3b1b0 [term 11 FOLLOWER]: Refusing update from remote peer 67bf7defb0654350846098210079d01d: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
23:59:36.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:36.460376 22618 consensus_queue.cc:1035] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d [LEADER]: Connected to new peer: Peer: permanent_uuid: "df2cc700bf114fb3af3f4fb7fdd3b1b0" member_type: VOTER last_known_addr { host: "127.19.197.2" port: 45931 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
23:59:38.914 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 444a22f6bc684186b6cc99f4d034b72b for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820' with partition [<start>, <end>)
23:59:38.915 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=444a22f6bc684186b6cc99f4d034b72b@[67bf7defb0654350846098210079d01d(127.19.197.3:33657)[L],6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839),df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)]}]
23:59:38.915 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=444a22f6bc684186b6cc99f4d034b72b, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=4429), Traces: [0ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [2ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [9ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [10ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [11ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [29ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [30ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [32ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [32ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [69ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [70ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [71ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [71ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [189ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [190ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [192ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [192ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [386ms] refreshing cache from master, [387ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [389ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [391ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [392ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [392ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [466ms] refreshing cache from master, [467ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [469ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, [471ms] sending RPC to server 6a4687af738441b284ae2313d7276cbf, [472ms] received response from server 6a4687af738441b284ae2313d7276cbf: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [472ms] delaying RPC due to: Network error: Failed to connect to peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839): Connection refused: /127.19.197.1:36839, [4427ms] refreshing cache from master, [4427ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.197.62:35395, [4429ms] Sub RPC GetTableLocations: received response from server master-127.19.197.62:35395: OK, deferred=Deferred@1183255261(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/707911881@5149ffdc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580) -> (continuation of Deferred@1735342974 after retry RPC@962477617) -> (continuation of Deferred@1685408287 after retry RPC@474979577) -> (continuation of Deferred@573960483 after retry RPC@1115494856), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@241b1c09 -> wakeup thread Time-limited test -> (continuation of Deferred@147064619 after retry RPC@1930066660) -> (continuation of Deferred@443230362 after retry RPC@974568553) -> (continuation of Deferred@1593499528 after retry RPC@819840580) -> (continuation of Deferred@1735342974 after retry RPC@962477617) -> (continuation of Deferred@1685408287 after retry RPC@474979577) -> (continuation of Deferred@573960483 after retry RPC@1115494856))) after lookup
23:59:38.916 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] connecting to peer
23:59:38.917 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] Successfully connected to peer
23:59:38.933 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7d6631c6, L:/127.0.0.1:38372 - R:/127.19.197.3:33657] using TOKEN/null
23:59:38.954 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.19.197.1:36839
23:59:38.954 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:38.955329 20244 external_mini_cluster.cc:1326] Running /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task84Dvx3/build/asan/bin/kudu
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:59:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:59:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.163405 21934 consensus_peers.cc:487] T 444a22f6bc684186b6cc99f4d034b72b P 67bf7defb0654350846098210079d01d -> Peer 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839): Couldn't send request to peer 6a4687af738441b284ae2313d7276cbf. Status: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:59:39.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.279126 22661 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:59:39.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.279459 22661 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:59:39.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.279577 22661 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:59:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.287050 22661 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:59:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.287292 22661 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.197.1
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.297117 22661 tablet_server_runner.cc:78] Tablet server non-default flags:
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.197.20:43221
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.197.1:36839
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.197.1
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35163
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.197.62:35395,127.19.197.61:35737,127.19.197.60:43759
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:59:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.197.1
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/logs
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Jul 2026 22:40:22 UTC on ccycloud.quasar-znwdty.root.comops.site
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4492328
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
23:59:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.298774 22661 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:59:39.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.300570 22661 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:59:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.314270 22671 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.314358 22669 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.314371 22668 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:59:39.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.319679 22661 server_base.cc:1034] running on GCE node
23:59:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.320317 22661 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:59:39.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.321290 22661 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:59:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.322547 22661 hybrid_clock.cc:648] HybridClock initialized: now 1783468779322454 us; error 54 us; skew 500 ppm
23:59:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.322932 22661 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:59:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.329345 22661 webserver.cc:460] Webserver started at http://127.19.197.1:35163/ using document root <none> and password file <none>
23:59:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.330277 22661 fs_manager.cc:359] Metadata directory not provided
23:59:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.330435 22661 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:59:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.336252 22661 fs_manager.cc:709] Time spent opening directory manager: real 0.003s	user 0.001s	sys 0.004s
23:59:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.339372 22677 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:59:39.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.340744 22661 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
23:59:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.340876 22661 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data,/tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a4687af738441b284ae2313d7276cbf"
23:59:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:59:04 on dist-test-slave-x4hg"
23:59:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.341781 22661 fs_report.cc:383] FS layout report
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/wal
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/data
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:59:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:59:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.361824 22661 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
23:59:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.362650 22661 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:59:39.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.362954 22661 kserver.cc:163] Server-wide thread pool size limit: 3276
23:59:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.363940 22661 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:59:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.365862 22684 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
23:59:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.371114 22661 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:59:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.371243 22661 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
23:59:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.371402 22661 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
23:59:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.374408 22661 ts_tablet_manager.cc:610] Registered 1 tablets
23:59:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.374511 22661 ts_tablet_manager.cc:589] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
23:59:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.374755 22684 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap starting.
23:59:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.403990 22684 log.cc:824] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Log is configured to *not* fsync() on all Append() calls
23:59:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.424484 22661 rpc_server.cc:307] RPC server started. Bound to: 127.19.197.1:36839
23:59:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.424502 22791 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.197.1:36839 every 8 connection(s)
23:59:39.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.425675 22661 server_base.cc:1166] Dumped server information to /tmp/dist-test-task84Dvx3/test-tmp/mini-kudu-cluster6848220396874853285/ts-0/data/info.pb
23:59:39.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.431346 20244 external_mini_cluster.cc:1388] Started /tmp/dist-test-task84Dvx3/build/asan/bin/kudu as pid 22661
23:59:39.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:59:39.449 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:39.458 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468745820, tablet=444a22f6bc684186b6cc99f4d034b72b, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:59:39.464 [DEBUG - main] (Connection.java:575) [peer master-127.19.197.61:35737(127.19.197.61:35737)] disconnecting while in state READY
23:59:39.466 [DEBUG - main] (Connection.java:575) [peer master-127.19.197.62:35395(127.19.197.62:35395)] disconnecting while in state READY
23:59:39.466 [DEBUG - main] (Connection.java:575) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] disconnecting while in state READY
23:59:39.466 [DEBUG - main] (Connection.java:575) [peer df2cc700bf114fb3af3f4fb7fdd3b1b0(127.19.197.2:45931)] disconnecting while in state TERMINATED
23:59:39.467 [DEBUG - main] (Connection.java:575) [peer 6a4687af738441b284ae2313d7276cbf(127.19.197.1:36839)] disconnecting while in state TERMINATED
23:59:39.467 [DEBUG - main] (Connection.java:575) [peer master-127.19.197.60:43759(127.19.197.60:43759)] disconnecting while in state READY
23:59:39.471 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.19.197.60:43759(127.19.197.60:43759)] handling channelInactive
23:59:39.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.197.60:43759(127.19.197.60:43759)] cleaning up while in state READY due to: connection closed
23:59:39.471 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:59:39.472 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.197.61:35737(127.19.197.61:35737)] handling channelInactive
23:59:39.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.197.61:35737(127.19.197.61:35737)] cleaning up while in state READY due to: connection closed
23:59:39.473 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.197.62:35395(127.19.197.62:35395)] handling channelInactive
23:59:39.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.197.62:35395(127.19.197.62:35395)] cleaning up while in state READY due to: connection closed
23:59:39.473 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] handling channelInactive
23:59:39.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 67bf7defb0654350846098210079d01d(127.19.197.3:33657)] cleaning up while in state READY due to: connection closed
23:59:39.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.477072 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 22661
23:59:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.518764 22794 heartbeater.cc:347] Connected to a master server at 127.19.197.61:35737
23:59:39.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.519074 22794 heartbeater.cc:464] Registering TS with master...
23:59:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.519907 22794 heartbeater.cc:511] Master 127.19.197.61:35737 requested a full tablet report, sending...
23:59:39.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.518724 22793 heartbeater.cc:347] Connected to a master server at 127.19.197.62:35395
23:59:39.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.521394 22793 heartbeater.cc:464] Registering TS with master...
23:59:39.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.521831 22793 heartbeater.cc:511] Master 127.19.197.62:35395 requested a full tablet report, sending...
23:59:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.522413 20357 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.523113 20286 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.523550 22792 heartbeater.cc:347] Connected to a master server at 127.19.197.60:43759
23:59:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.523660 22792 heartbeater.cc:464] Registering TS with master...
23:59:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.524062 22792 heartbeater.cc:511] Master 127.19.197.60:43759 requested a full tablet report, sending...
23:59:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.524595 20286 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.197.1:40277
23:59:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.525341 20428 ts_manager.cc:194] Re-registered known tserver with Master: 6a4687af738441b284ae2313d7276cbf (127.19.197.1:36839)
23:59:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.528358 22684 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:59:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.529177 22684 tablet_bootstrap.cc:492] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Bootstrap complete.
23:59:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.530848 22684 ts_tablet_manager.cc:1397] T 444a22f6bc684186b6cc99f4d034b72b P 6a4687af738441b284ae2313d7276cbf: Time spent bootstrapping tablet: real 0.156s	user 0.058s	sys 0.024s
23:59:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.530870 22787 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:39.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.623204 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 22661
23:59:39.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.645329 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 22471
23:59:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:59:39.646498 22662 negotiation.cc:337] Failed RPC negotiation. Trace:
23:59:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:39.633152 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.19.197.1:36839 (local address 127.19.197.3:52149)
23:59:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:39.633419 (+   267us) negotiation.cc:107] Waiting for socket to connect
23:59:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:39.633438 (+    19us) client_negotiation.cc:174] Beginning negotiation
23:59:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:39.633565 (+   127us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
23:59:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:59:39.645378 (+ 11813us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.19.197.1:36839: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:59:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":137}
23:59:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.653343 22597 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.758315 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 22471
23:59:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.778946 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 21917
23:59:39.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.787012 22043 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.911903 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 21917
23:59:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.932806 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 20255
23:59:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:39.933976 20316 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:40.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:40.053867 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 20255
23:59:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:40.076053 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 20323
23:59:40.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:40.077107 20387 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:40.192222 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 20323
23:59:40.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:40.213198 20244 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task84Dvx3/build/asan/bin/kudu pid 20394
23:59:40.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:40.214354 20458 generic_service.cc:196] Checking for leaks (request via RPC)
23:59:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:59:40.324983 20244 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task84Dvx3/build/asan/bin/kudu with pid 20394
23:59:40.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:59:40Z chronyd exiting

Time: 39.447

OK (1 test)

