JUnit version 4.13.2
.14:30:03.418 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783521003411
14:30:03.484 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
14:30:03.526 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:30:03.542 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskksXv_G/test-tmp
14:30:03.569 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:30:03.572 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskksXv_G/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:30:03.759 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

14:30:03.943 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
14:30:03.944 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

14:30:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:03.948369 27463 mini_oidc.cc:150] Starting JWKS server
14:30:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:03.951922 27463 webserver.cc:533] Webserver started at http://127.0.0.1:38243/ using document root <none> and password file <none>
14:30:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:03.952433 27463 mini_oidc.cc:186] Starting OIDC Discovery server
14:30:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:03.954718 27463 webserver.cc:533] Webserver started at http://127.0.0.1:43915/ using document root <none> and password file <none>
14:30:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T14:30:04Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:30:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T14:30:04Z Disabled control of system clock
14:30:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:04.014294 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/wal
14:30:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data
14:30:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/logs
14:30:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data/info.pb
14:30:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.254:45405
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.254
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:30:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:30:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38243/jwks.json with env {}
14:30:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.326843 27476 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.327560 27476 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.328069 27476 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.361119 27476 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:30:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.361419 27476 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.361615 27476 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:30:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.361850 27476 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:30:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.362068 27476 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:30:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.362244 27476 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38243/jwks.json
14:30:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:04.399573 27476 master_runner.cc:387] Master server non-default flags:
14:30:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data
14:30:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/wal
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.254:45405
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:30:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38243/jwks.json
14:30:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data/info.pb
14:30:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.254
14:30:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:04.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:04.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/logs
14:30:04.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:04.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:30:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:04.401027 27476 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:04.402796 27476 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.415712 27482 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:04.415699 27483 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.539279 27485 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:05.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.542533 27484 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1123 milliseconds
14:30:05.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.542709 27476 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.128s	user 0.348s	sys 0.778s
14:30:05.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.543085 27476 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.129s	user 0.348s	sys 0.778s
14:30:05.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.543376 27476 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.544917 27476 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.548257 27476 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.550261 27476 hybrid_clock.cc:648] HybridClock initialized: now 1783521005550223 us; error 36 us; skew 500 ppm
14:30:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.551447 27476 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.560762 27476 webserver.cc:533] Webserver started at http://127.26.209.254:33553/ using document root <none> and password file <none>
14:30:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.562228 27476 fs_manager.cc:362] Metadata directory not provided
14:30:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.562558 27476 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.563159 27476 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:30:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.569859 27476 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data/instance:
14:30:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bc69230df0e4caca087586a7fafad3a"
14:30:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:05 on dist-test-slave-rt66"
14:30:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.571558 27476 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/wal/instance:
14:30:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bc69230df0e4caca087586a7fafad3a"
14:30:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:05 on dist-test-slave-rt66"
14:30:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.582542 27476 fs_manager.cc:696] Time spent creating directory manager: real 0.010s	user 0.006s	sys 0.006s
14:30:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.590178 27492 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.592002 27476 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.003s
14:30:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.592329 27476 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/wal
14:30:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bc69230df0e4caca087586a7fafad3a"
14:30:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:05 on dist-test-slave-rt66"
14:30:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.592650 27476 fs_report.cc:389] FS layout report
14:30:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/wal
14:30:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/wal
14:30:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data/data
14:30:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.657640 27476 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.659276 27476 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.659804 27476 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.729202 27476 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.254:45405
14:30:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.729238 27543 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.254:45405 every 8 connection(s)
14:30:05.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.731891 27476 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/data/info.pb
14:30:05.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.737639 27544 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:30:05.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.742657 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 27476
14:30:05.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.743136 27463 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-0/wal/instance
14:30:05.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.746644 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:05.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:05.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/wal
14:30:05.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data
14:30:05.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/logs
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data/info.pb
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:30:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:30:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:30:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.253:34165
14:30:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.253
14:30:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:30:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:30:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38243/jwks.json with env {}
14:30:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.748188 27544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } has no permanent_uuid. Determining permanent_uuid...
14:30:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.781554 27544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } has no permanent_uuid. Determining permanent_uuid...
14:30:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.784137 27496 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.788584 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.253:34165: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.838150 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } attempt: 1
14:30:05.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.843080 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.253:34165: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:05.911522 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } attempt: 2
14:30:05.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:05.916903 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.253:34165: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:06.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:06.008399 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } attempt: 3
14:30:06.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.013301 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.253:34165: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.063611 27546 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.064285 27546 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.065038 27546 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:06.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.098415 27546 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:30:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.098773 27546 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.099088 27546 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:30:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.099402 27546 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:30:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.099756 27546 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:30:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.100028 27546 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38243/jwks.json
14:30:06.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:06.138238 27546 master_runner.cc:387] Master server non-default flags:
14:30:06.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:06.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:06.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data
14:30:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/wal
14:30:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:30:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:30:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.253:34165
14:30:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:30:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:30:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38243/jwks.json
14:30:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data/info.pb
14:30:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.253
14:30:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/logs
14:30:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:30:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:06.139994 27546 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:06.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:06.142424 27546 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.155211 27555 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:06.156697 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } attempt: 4
14:30:06.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.162299 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.253:34165: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:06.461875 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } attempt: 5
14:30:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.468103 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.253:34165: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.015605 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } attempt: 6
14:30:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.023983 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.253:34165: Network error: Client connection negotiation failed: client connection to 127.26.209.253:34165: connect: Connection refused (error 111)
14:30:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.156158 27556 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:06.161384 27558 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.300350 27557 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:30:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.300454 27546 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.305650 27546 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:07.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.308127 27546 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.309496 27546 hybrid_clock.cc:648] HybridClock initialized: now 1783521007309462 us; error 49 us; skew 500 ppm
14:30:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.310388 27546 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.316680 27546 webserver.cc:533] Webserver started at http://127.26.209.253:36827/ using document root <none> and password file <none>
14:30:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.317705 27546 fs_manager.cc:362] Metadata directory not provided
14:30:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.317921 27546 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.318503 27546 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:30:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.322789 27546 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data/instance:
14:30:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "800b69f2fe4d4ca2b62da4132a3e3514"
14:30:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:07 on dist-test-slave-rt66"
14:30:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.324363 27546 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/wal/instance:
14:30:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "800b69f2fe4d4ca2b62da4132a3e3514"
14:30:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:07 on dist-test-slave-rt66"
14:30:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.332383 27546 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.000s
14:30:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.338626 27567 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.339964 27546 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
14:30:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.340339 27546 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/wal
14:30:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "800b69f2fe4d4ca2b62da4132a3e3514"
14:30:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:07 on dist-test-slave-rt66"
14:30:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.340828 27546 fs_report.cc:389] FS layout report
14:30:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/wal
14:30:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/wal
14:30:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data/data
14:30:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.401332 27546 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.402838 27546 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.403241 27546 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.478662 27546 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.253:34165
14:30:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.478838 27618 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.253:34165 every 8 connection(s)
14:30:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.481614 27546 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/data/info.pb
14:30:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.483681 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 27546
14:30:07.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.484148 27463 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-1/wal/instance
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.487875 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/wal
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/logs
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data/info.pb
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.252:41417
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.252
14:30:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38243/jwks.json with env {}
14:30:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.490473 27619 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:30:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.507673 27619 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } has no permanent_uuid. Determining permanent_uuid...
14:30:07.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.534310 27619 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } has no permanent_uuid. Determining permanent_uuid...
14:30:07.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.552284 27619 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } has no permanent_uuid. Determining permanent_uuid...
14:30:07.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.555285 27571 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.560269 27619 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.609910 27619 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 1
14:30:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.615360 27619 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.683917 27619 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 2
14:30:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.689320 27619 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:07.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.780956 27619 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 3
14:30:07.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.786768 27619 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.816721 27621 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.817391 27621 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.817858 27621 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.851289 27621 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:30:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.851585 27621 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.851783 27621 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:30:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.851974 27621 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:30:07.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.852166 27621 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:30:07.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.852356 27621 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38243/jwks.json
14:30:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.889323 27621 master_runner.cc:387] Master server non-default flags:
14:30:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data
14:30:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/wal
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.252:41417
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38243/jwks.json
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data/info.pb
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.252
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/logs
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.890686 27621 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.892752 27621 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.904233 27631 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:07.930258 27619 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 4
14:30:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.937407 27619 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.084671 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } attempt: 7
14:30:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.128661 27544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } has no permanent_uuid. Determining permanent_uuid...
14:30:08.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.151937 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.210983 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 1
14:30:08.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.217949 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.236888 27619 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 5
14:30:08.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.245862 27619 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.299429 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 2
14:30:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.306578 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.392028 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 3
14:30:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.399173 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.539682 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 4
14:30:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.547262 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.793354 27619 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 6
14:30:08.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:08.830736 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 5
14:30:08.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.839047 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:08.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:08.840091 27619 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:09.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:07.905499 27632 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:09.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.285753 27633 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1377 milliseconds
14:30:09.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.286118 27634 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:09.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.288487 27621 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.384s	user 0.542s	sys 0.827s
14:30:09.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.288839 27621 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.384s	user 0.546s	sys 0.827s
14:30:09.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.289126 27621 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.290925 27621 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.293473 27621 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.294824 27621 hybrid_clock.cc:648] HybridClock initialized: now 1783521009294757 us; error 82 us; skew 500 ppm
14:30:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.295698 27621 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.302677 27621 webserver.cc:533] Webserver started at http://127.26.209.252:43871/ using document root <none> and password file <none>
14:30:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.303669 27621 fs_manager.cc:362] Metadata directory not provided
14:30:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.303905 27621 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.304440 27621 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:30:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.308627 27621 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data/instance:
14:30:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8784e1234544888af66b1ef352c220f"
14:30:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:09 on dist-test-slave-rt66"
14:30:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.309931 27621 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/wal/instance:
14:30:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8784e1234544888af66b1ef352c220f"
14:30:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:09 on dist-test-slave-rt66"
14:30:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.317914 27621 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.002s
14:30:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.323962 27647 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:09.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.325172 27621 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.001s
14:30:09.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.325502 27621 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/wal
14:30:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8784e1234544888af66b1ef352c220f"
14:30:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:09 on dist-test-slave-rt66"
14:30:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.325873 27621 fs_report.cc:389] FS layout report
14:30:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/wal
14:30:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/wal
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data/data
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:09.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.391541 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 6
14:30:09.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.398536 27544 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.209.252:41417: Network error: Client connection negotiation failed: client connection to 127.26.209.252:41417: connect: Connection refused (error 111)
14:30:09.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.399258 27621 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:09.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.400653 27621 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:09.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.401201 27621 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:09.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.473946 27621 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.252:41417
14:30:09.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.473999 27700 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.252:41417 every 8 connection(s)
14:30:09.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.477504 27621 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/data/info.pb
14:30:09.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.484817 27701 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:30:09.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.487185 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 27621
14:30:09.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.487573 27463 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/master-2/wal/instance
14:30:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.497043 27701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } has no permanent_uuid. Determining permanent_uuid...
14:30:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.521708 27701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } has no permanent_uuid. Determining permanent_uuid...
14:30:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.534088 27701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } has no permanent_uuid. Determining permanent_uuid...
14:30:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.550729 27701 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f: Bootstrap starting.
14:30:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.557231 27701 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f: Neither blocks nor log segments found. Creating new log.
14:30:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.559240 27701 log.cc:826] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f: Log is configured to *not* fsync() on all Append() calls
14:30:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.564185 27701 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f: No bootstrap required, opened a new log
14:30:09.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.588071 27701 raft_consensus.cc:359] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.589560 27701 raft_consensus.cc:385] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:30:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.589879 27701 raft_consensus.cc:740] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8784e1234544888af66b1ef352c220f, State: Initialized, Role: FOLLOWER
14:30:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.590854 27701 consensus_queue.cc:260] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:09.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.593384 27711 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:09.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.594208 27711 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: This master's current role is: FOLLOWER
14:30:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.594802 27701 sys_catalog.cc:565] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: configured and running, proceeding with master startup.
14:30:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.635437 27722 catalog_manager.cc:1594] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:30:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.636478 27722 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:30:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.617851 27621 master_runner.cc:187] Error getting master registration for 127.26.209.254:45405: OK, instance_id { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" instance_seqno: 1783521005709982 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:30:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.638634 27621 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:30:09.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.656901 27655 tablet.cc:2406] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:30:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.900559 27619 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 7
14:30:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.921744 27619 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: Bootstrap starting.
14:30:09.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.927528 27619 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: Neither blocks nor log segments found. Creating new log.
14:30:09.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.929625 27619 log.cc:826] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: Log is configured to *not* fsync() on all Append() calls
14:30:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.933941 27619 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: No bootstrap required, opened a new log
14:30:09.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.953536 27619 raft_consensus.cc:359] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:09.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.954429 27619 raft_consensus.cc:385] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:30:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.954705 27619 raft_consensus.cc:740] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 800b69f2fe4d4ca2b62da4132a3e3514, State: Initialized, Role: FOLLOWER
14:30:09.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.955458 27619 consensus_queue.cc:260] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.958225 27726 sys_catalog.cc:455] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.959028 27726 sys_catalog.cc:458] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [sys.catalog]: This master's current role is: FOLLOWER
14:30:09.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.959942 27619 sys_catalog.cc:565] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [sys.catalog]: configured and running, proceeding with master startup.
14:30:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.991382 27737 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:30:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:09.992354 27737 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:30:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:09.986225 27546 master_runner.cc:187] Error getting master registration for 127.26.209.254:45405: OK, instance_id { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" instance_seqno: 1783521005709982 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:30:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.008329 27546 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:30:10.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.075627 27711 raft_consensus.cc:493] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:30:10.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.076129 27711 raft_consensus.cc:515] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:10.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.078564 27711 leader_election.cc:290] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9bc69230df0e4caca087586a7fafad3a (127.26.209.254:45405), 800b69f2fe4d4ca2b62da4132a3e3514 (127.26.209.253:34165)
14:30:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.080497 27594 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f8784e1234544888af66b1ef352c220f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" is_pre_election: true
14:30:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.081385 27594 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f8784e1234544888af66b1ef352c220f in term 0.
14:30:10.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.083076 27651 leader_election.cc:304] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 800b69f2fe4d4ca2b62da4132a3e3514, f8784e1234544888af66b1ef352c220f; no voters: 
14:30:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.084767 27711 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Leader pre-election won for term 1
14:30:10.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.085131 27711 raft_consensus.cc:493] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:30:10.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.085455 27711 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 0 FOLLOWER]: Advancing to term 1
14:30:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.096064 27711 raft_consensus.cc:515] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.104039 27711 leader_election.cc:290] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [CANDIDATE]: Term 1 election: Requested vote from peers 9bc69230df0e4caca087586a7fafad3a (127.26.209.254:45405), 800b69f2fe4d4ca2b62da4132a3e3514 (127.26.209.253:34165)
14:30:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.105160 27594 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f8784e1234544888af66b1ef352c220f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "800b69f2fe4d4ca2b62da4132a3e3514"
14:30:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.105700 27594 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [term 0 FOLLOWER]: Advancing to term 1
14:30:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.110962 27594 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f8784e1234544888af66b1ef352c220f in term 1.
14:30:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.112016 27651 leader_election.cc:304] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 800b69f2fe4d4ca2b62da4132a3e3514, f8784e1234544888af66b1ef352c220f; no voters: 
14:30:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.113919 27711 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 1 FOLLOWER]: Leader election won for term 1
14:30:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.116312 27711 raft_consensus.cc:697] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [term 1 LEADER]: Becoming Leader. State: Replica: f8784e1234544888af66b1ef352c220f, State: Running, Role: LEADER
14:30:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.117612 27711 consensus_queue.cc:237] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.125141 27739 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f8784e1234544888af66b1ef352c220f. Latest consensus state: current_term: 1 leader_uuid: "f8784e1234544888af66b1ef352c220f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.125874 27739 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: This master's current role is: LEADER
14:30:10.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.137734 27741 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
14:30:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.145512 27741 catalog_manager.cc:1520] Initializing Kudu cluster ID...
14:30:10.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:10.162210 27573 tablet.cc:2406] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:30:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.163074 27594 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [term 1 FOLLOWER]: Refusing update from remote peer f8784e1234544888af66b1ef352c220f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.165412 27739 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [LEADER]: Connected to new peer: Peer: permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:30:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.197075 27742 mvcc.cc:204] Tried to move back new op lower bound from 7305302057608347648 to 7305302057452945408. Current Snapshot: MvccSnapshot[applied={T|T < 7305302057608347648}]
14:30:10.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.211721 27726 sys_catalog.cc:455] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f8784e1234544888af66b1ef352c220f. Latest consensus state: current_term: 1 leader_uuid: "f8784e1234544888af66b1ef352c220f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:10.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.218777 27726 sys_catalog.cc:458] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [sys.catalog]: This master's current role is: FOLLOWER
14:30:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.225750 27711 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f8784e1234544888af66b1ef352c220f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.226491 27711 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: This master's current role is: LEADER
14:30:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.228113 27741 catalog_manager.cc:1383] Generated new cluster ID: 57891e16af634a348f62fc3e06f03d9e
14:30:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.228351 27741 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
14:30:10.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.253281 27746 sys_catalog.cc:455] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f8784e1234544888af66b1ef352c220f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:10.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.255916 27746 sys_catalog.cc:458] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514 [sys.catalog]: This master's current role is: FOLLOWER
14:30:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.269707 27741 catalog_manager.cc:1406] Generated new certificate authority record
14:30:10.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.280426 27741 catalog_manager.cc:1540] Loading token signing keys...
14:30:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.312568 27741 catalog_manager.cc:6070] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f: Generated new TSK 0
14:30:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.313764 27741 catalog_manager.cc:1550] Initializing in-progress tserver states...
14:30:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.339970 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:0
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:30:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.431967 27544 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } attempt: 7
14:30:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.510152 27544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: Bootstrap starting.
14:30:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.518507 27544 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: Neither blocks nor log segments found. Creating new log.
14:30:10.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.539076 27544 log.cc:826] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: Log is configured to *not* fsync() on all Append() calls
14:30:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.564635 27544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: No bootstrap required, opened a new log
14:30:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.629240 27544 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.630645 27544 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:30:10.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.636900 27544 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9bc69230df0e4caca087586a7fafad3a, State: Initialized, Role: FOLLOWER
14:30:10.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.638432 27544 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } }
14:30:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.642370 27753 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:10.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.644479 27753 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [sys.catalog]: This master's current role is: FOLLOWER
14:30:10.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.651346 27544 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [sys.catalog]: configured and running, proceeding with master startup.
14:30:10.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.651525 27518 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f8784e1234544888af66b1ef352c220f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9bc69230df0e4caca087586a7fafad3a"
14:30:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.652259 27518 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [term 0 FOLLOWER]: Advancing to term 1
14:30:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.658746 27519 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f8784e1234544888af66b1ef352c220f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9bc69230df0e4caca087586a7fafad3a" is_pre_election: true
14:30:10.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.665565 27518 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f8784e1234544888af66b1ef352c220f in term 1.
14:30:10.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.667685 27517 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [term 1 FOLLOWER]: Refusing update from remote peer f8784e1234544888af66b1ef352c220f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:30:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:10.673316 27498 tablet.cc:2406] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:30:10.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.679324 27711 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [LEADER]: Connected to new peer: Peer: permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:30:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.778530 27753 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [sys.catalog]: SysCatalogTable state changed. Reason: New leader f8784e1234544888af66b1ef352c220f. Latest consensus state: current_term: 1 leader_uuid: "f8784e1234544888af66b1ef352c220f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.779246 27753 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [sys.catalog]: This master's current role is: FOLLOWER
14:30:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.780400 27760 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f8784e1234544888af66b1ef352c220f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.781157 27760 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a [sys.catalog]: This master's current role is: FOLLOWER
14:30:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.789110 27769 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:30:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:10.956332 27749 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:10.956907 27749 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:10.957549 27749 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:10.991214 27749 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:10.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:10.992188 27749 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.193
14:30:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.999244 27737 catalog_manager.cc:1295] Loaded cluster ID: 57891e16af634a348f62fc3e06f03d9e
14:30:11.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:10.999929 27737 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: loading cluster ID for follower catalog manager: success
14:30:11.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.007629 27737 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: acquiring CA information for follower catalog manager: success
14:30:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.013568 27737 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 800b69f2fe4d4ca2b62da4132a3e3514: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:30:11.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.037029 27749 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:11.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:11.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:0
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.038748 27749 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.041018 27749 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:11.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:11.057130 27775 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.189692 27779 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f8784e1234544888af66b1ef352c220f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc69230df0e4caca087586a7fafad3a" member_type: VOTER last_known_addr { host: "127.26.209.254" port: 45405 } } peers { permanent_uuid: "800b69f2fe4d4ca2b62da4132a3e3514" member_type: VOTER last_known_addr { host: "127.26.209.253" port: 34165 } } peers { permanent_uuid: "f8784e1234544888af66b1ef352c220f" member_type: VOTER last_known_addr { host: "127.26.209.252" port: 41417 } } }
14:30:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.193424 27779 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8784e1234544888af66b1ef352c220f [sys.catalog]: This master's current role is: LEADER
14:30:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.810642 27769 catalog_manager.cc:1295] Loaded cluster ID: 57891e16af634a348f62fc3e06f03d9e
14:30:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.811595 27769 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: loading cluster ID for follower catalog manager: success
14:30:11.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.820021 27769 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: acquiring CA information for follower catalog manager: success
14:30:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:11.826203 27769 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 9bc69230df0e4caca087586a7fafad3a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:30:13.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:12.459429 27774 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 27749
14:30:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.063959 27749 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.005s	user 0.623s	sys 1.024s
14:30:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.065783 27749 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.007s	user 0.623s	sys 1.025s
14:30:13.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:11.060593 27776 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.079854 27749 server_base.cc:1061] running on GCE node
14:30:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.079885 27786 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.081835 27749 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:13.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.084578 27749 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.086057 27749 hybrid_clock.cc:648] HybridClock initialized: now 1783521013085877 us; error 194 us; skew 500 ppm
14:30:13.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.087100 27749 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.095012 27749 webserver.cc:533] Webserver started at http://127.26.209.193:45851/ using document root <none> and password file <none>
14:30:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.096206 27749 fs_manager.cc:362] Metadata directory not provided
14:30:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.096494 27749 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.097050 27749 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:30:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.102981 27749 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/instance:
14:30:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:13 on dist-test-slave-rt66"
14:30:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.104236 27749 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal/instance:
14:30:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:13 on dist-test-slave-rt66"
14:30:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.111675 27749 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.005s
14:30:13.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.118440 27791 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.119809 27749 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.003s
14:30:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.120258 27749 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:13 on dist-test-slave-rt66"
14:30:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.120632 27749 fs_report.cc:389] FS layout report
14:30:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/data
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:13.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.172438 27749 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:13.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.174293 27749 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.174825 27749 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:13.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.177901 27749 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:13.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.182296 27749 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:30:13.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.182510 27749 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:30:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.182806 27749 ts_tablet_manager.cc:616] Registered 0 tablets
14:30:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.182988 27749 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:30:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.349699 27749 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.193:36347
14:30:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.349807 27903 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.193:36347 every 8 connection(s)
14:30:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.352288 27749 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.360090 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 27749
14:30:13.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.360545 27463 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal/instance
14:30:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.382181 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:0
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.412707 27906 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.413470 27906 heartbeater.cc:461] Registering TS with master...
14:30:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.415060 27906 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.418963 27508 ts_manager.cc:194] Registered new tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.419045 27904 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.419397 27904 heartbeater.cc:461] Registering TS with master...
14:30:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.420308 27904 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.420697 27905 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.421093 27905 heartbeater.cc:461] Registering TS with master...
14:30:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.421998 27905 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.424049 27666 ts_manager.cc:194] Registered new tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.426436 27584 ts_manager.cc:194] Registered new tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.426764 27666 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.193:46879
14:30:13.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.736342 27914 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:13.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.737299 27914 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:13.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.738026 27914 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:13.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.784703 27914 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:13.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.785575 27914 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.194
14:30:13.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.821327 27914 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:0
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:13.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.822654 27914 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:13.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:13.824220 27914 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:13.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.838120 27920 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:14.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.430595 27904 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.844530 27923 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:13.841300 27921 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:14.973292 27922 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:30:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.973510 27914 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.977289 27914 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.979779 27914 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.981192 27914 hybrid_clock.cc:648] HybridClock initialized: now 1783521014981159 us; error 49 us; skew 500 ppm
14:30:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.981957 27914 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.988467 27914 webserver.cc:533] Webserver started at http://127.26.209.194:35263/ using document root <none> and password file <none>
14:30:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.989398 27914 fs_manager.cc:362] Metadata directory not provided
14:30:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.989593 27914 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.990029 27914 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:30:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.994594 27914 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/instance:
14:30:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:14 on dist-test-slave-rt66"
14:30:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:14.995683 27914 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal/instance:
14:30:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:14 on dist-test-slave-rt66"
14:30:15.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.003472 27914 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.004s
14:30:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.009507 27932 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:15.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.010761 27914 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
14:30:15.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.011129 27914 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:15.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:15.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:14 on dist-test-slave-rt66"
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.011518 27914 fs_report.cc:389] FS layout report
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/data
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.065830 27914 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.067276 27914 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.067674 27914 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.070420 27914 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.074870 27914 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:30:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.075078 27914 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:30:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.075269 27914 ts_tablet_manager.cc:616] Registered 0 tablets
14:30:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.075398 27914 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:30:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.206418 27914 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.194:44581
14:30:15.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.206527 28044 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.194:44581 every 8 connection(s)
14:30:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.209012 27914 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.218724 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 27914
14:30:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.219336 27463 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal/instance
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.229687 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:0
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:15.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:15.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:15.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.241799 28045 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.242684 28045 heartbeater.cc:461] Registering TS with master...
14:30:15.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.244244 28045 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.247435 28046 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.247599 27666 ts_manager.cc:194] Registered new tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:15.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.248169 28046 heartbeater.cc:461] Registering TS with master...
14:30:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.249289 27666 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.194:59129
14:30:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.249439 28046 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.252091 27584 ts_manager.cc:194] Registered new tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.255170 28047 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.255533 28047 heartbeater.cc:461] Registering TS with master...
14:30:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.256213 28047 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.258675 27508 ts_manager.cc:194] Registered new tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:15.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:15.557135 28054 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:15.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:15.557600 28054 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:15.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:15.558007 28054 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:15.592829 28054 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:15.593608 28054 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.195
14:30:15.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.627991 28054 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:15.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:15.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:0
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:15.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.629347 28054 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:15.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:15.631199 28054 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:15.643455 28060 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:16.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.254916 28045 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:16.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:15.644737 28061 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:16.864915 28063 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:16.866562 28062 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1218 milliseconds
14:30:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:16.867373 28054 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.224s	user 0.506s	sys 0.702s
14:30:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:16.867627 28054 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.224s	user 0.506s	sys 0.702s
14:30:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.867913 28054 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:16.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.869118 28054 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:16.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.871373 28054 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.872750 28054 hybrid_clock.cc:648] HybridClock initialized: now 1783521016872715 us; error 43 us; skew 500 ppm
14:30:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.873680 28054 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.880906 28054 webserver.cc:533] Webserver started at http://127.26.209.195:39951/ using document root <none> and password file <none>
14:30:16.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.881863 28054 fs_manager.cc:362] Metadata directory not provided
14:30:16.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.882081 28054 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:16.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.882597 28054 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:30:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.887120 28054 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/instance:
14:30:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:16 on dist-test-slave-rt66"
14:30:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.888293 28054 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal/instance:
14:30:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:16 on dist-test-slave-rt66"
14:30:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.895789 28054 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.001s
14:30:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.901575 28071 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.902721 28054 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
14:30:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.903051 28054 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:16 on dist-test-slave-rt66"
14:30:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.903431 28054 fs_report.cc:389] FS layout report
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/data
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:16.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.982882 28054 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.984899 28054 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.985502 28054 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:16.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.989044 28054 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.994262 28054 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:30:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.994496 28054 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
14:30:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.994778 28054 ts_tablet_manager.cc:616] Registered 0 tablets
14:30:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:16.995008 28054 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
14:30:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.135190 28054 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.195:44447
14:30:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.135311 28183 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.195:44447 every 8 connection(s)
14:30:17.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.137904 28054 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:17.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.143663 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 28054
14:30:17.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.144156 27463 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal/instance
14:30:17.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.173170 28185 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:17.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.173703 28185 heartbeater.cc:461] Registering TS with master...
14:30:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.175035 28185 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:17.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.176381 28184 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.176708 28184 heartbeater.cc:461] Registering TS with master...
14:30:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.177680 28184 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.180181 28186 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.180480 28186 heartbeater.cc:461] Registering TS with master...
14:30:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.180570 27665 ts_manager.cc:194] Registered new tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.180754 27584 ts_manager.cc:194] Registered new tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.181416 28186 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.182330 27665 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.195:59577
14:30:17.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.184516 27508 ts_manager.cc:194] Registered new tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:17.187647 27463 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
14:30:17.200 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
14:30:17.201 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

14:30:17.210 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.26.209.254"
      port: 45405
    }
    bound_http_address {
      host: "127.26.209.254"
      port: 33553
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.26.209.253"
      port: 34165
    }
    bound_http_address {
      host: "127.26.209.253"
      port: 36827
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.26.209.252"
      port: 41417
    }
    bound_http_address {
      host: "127.26.209.252"
      port: 43871
    }
  }
}

14:30:17.213 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

14:30:17.217 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.26.209.193"
      port: 36347
    }
    bound_http_address {
      host: "127.26.209.193"
      port: 45851
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.26.209.194"
      port: 44581
    }
    bound_http_address {
      host: "127.26.209.194"
      port: 35263
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.26.209.195"
      port: 44447
    }
    bound_http_address {
      host: "127.26.209.195"
      port: 39951
    }
  }
}

14:30:17.217 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:30:17.401 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:30:17.464 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
14:30:17.674 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.209.252:41417(127.26.209.252:41417)] connecting to peer
14:30:17.725 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.209.253:34165(127.26.209.253:34165)] connecting to peer
14:30:17.741 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.209.254:45405(127.26.209.254:45405)] connecting to peer
14:30:17.745 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.209.253:34165(127.26.209.253:34165)] Successfully connected to peer
14:30:17.753 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.209.252:41417(127.26.209.252:41417)] Successfully connected to peer
14:30:17.753 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.26.209.254:45405(127.26.209.254:45405)] Successfully connected to peer
14:30:17.859 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.26.209.254
14:30:17.860 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.26.209.252
14:30:17.862 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.26.209.253
14:30:18.069 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x52f80f75, L:/127.0.0.1:44132 - R:/127.26.209.252:41417] using SASL/PLAIN
14:30:18.070 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x03f062be, L:/127.0.0.1:33406 - R:/127.26.209.253:34165] using SASL/PLAIN
14:30:18.070 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x5a48fec9, L:/127.0.0.1:35496 - R:/127.26.209.254:45405] using SASL/PLAIN
14:30:18.108 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:30:18.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=474), Traces: [0ms] refreshing cache from master, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.209.253:34165, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.209.254:45405, [449ms] Sub RPC ConnectToMaster: received response from server master-127.26.209.253:34165: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.26.209.254:45405: OK, [450ms] Sub RPC ConnectToMaster: received response from server master-127.26.209.252:41417: OK, deferred=null) after lookup
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.161680 27665 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44132:
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622"
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:30:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:30:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:30:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:30:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:18.166158 27665 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:30:18.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.186043 28184 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:18.262 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table a58e526f48e84e2bab5138c66df1299c
14:30:18.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.279629 27980 tablet_service.cc:1511] Processing CreateTablet for tablet 13b7e17c89d24bf7933f32c1ea7c621f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622 [id=a58e526f48e84e2bab5138c66df1299c]), partition=RANGE (key) PARTITION UNBOUNDED
14:30:18.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.279803 28119 tablet_service.cc:1511] Processing CreateTablet for tablet 13b7e17c89d24bf7933f32c1ea7c621f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622 [id=a58e526f48e84e2bab5138c66df1299c]), partition=RANGE (key) PARTITION UNBOUNDED
14:30:18.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.282240 27980 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 13b7e17c89d24bf7933f32c1ea7c621f. 1 dirs total, 0 dirs full, 0 dirs failed
14:30:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.283179 27839 tablet_service.cc:1511] Processing CreateTablet for tablet 13b7e17c89d24bf7933f32c1ea7c621f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622 [id=a58e526f48e84e2bab5138c66df1299c]), partition=RANGE (key) PARTITION UNBOUNDED
14:30:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.284296 28119 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 13b7e17c89d24bf7933f32c1ea7c621f. 1 dirs total, 0 dirs full, 0 dirs failed
14:30:18.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.285022 27839 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 13b7e17c89d24bf7933f32c1ea7c621f. 1 dirs total, 0 dirs full, 0 dirs failed
14:30:18.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.314386 28222 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap starting.
14:30:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.318785 28221 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap starting.
14:30:18.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.322996 28223 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap starting.
14:30:18.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.324183 28222 tablet_bootstrap.cc:654] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Neither blocks nor log segments found. Creating new log.
14:30:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.326653 28222 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Log is configured to *not* fsync() on all Append() calls
14:30:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.327234 28221 tablet_bootstrap.cc:654] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Neither blocks nor log segments found. Creating new log.
14:30:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.329387 28223 tablet_bootstrap.cc:654] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Neither blocks nor log segments found. Creating new log.
14:30:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.329558 28221 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Log is configured to *not* fsync() on all Append() calls
14:30:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.331693 28223 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Log is configured to *not* fsync() on all Append() calls
14:30:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.332566 28222 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: No bootstrap required, opened a new log
14:30:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.333165 28222 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent bootstrapping tablet: real 0.019s	user 0.013s	sys 0.004s
14:30:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.335103 28221 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: No bootstrap required, opened a new log
14:30:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.335633 28221 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent bootstrapping tablet: real 0.018s	user 0.004s	sys 0.009s
14:30:18.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.341475 28223 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: No bootstrap required, opened a new log
14:30:18.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.342125 28223 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent bootstrapping tablet: real 0.020s	user 0.015s	sys 0.000s
14:30:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.357139 28222 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.358068 28222 raft_consensus.cc:385] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:30:18.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.358356 28222 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 71108c6bdadf491b98e0d0b98c56682a, State: Initialized, Role: FOLLOWER
14:30:18.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.359292 28222 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.364428 28221 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.365607 28221 raft_consensus.cc:385] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:30:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.366061 28221 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Initialized, Role: FOLLOWER
14:30:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.367277 28221 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.369428 28222 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent starting tablet: real 0.036s	user 0.026s	sys 0.008s
14:30:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.374303 28223 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.375650 28223 raft_consensus.cc:385] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:30:18.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.376155 28223 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Initialized, Role: FOLLOWER
14:30:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.377493 28223 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:18.381052 27909 tablet.cc:2406] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:30:18.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.387818 28223 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent starting tablet: real 0.045s	user 0.028s	sys 0.011s
14:30:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.396245 28221 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent starting tablet: real 0.060s	user 0.027s	sys 0.025s
14:30:18.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:18.408623 28187 tablet.cc:2406] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:30:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:18.486953 28049 tablet.cc:2406] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:30:18.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.560307 28228 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:30:18.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.560969 28228 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.564899 28228 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:18.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.577051 27859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a" is_pre_election: true
14:30:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.578070 27859 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 0.
14:30:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.578989 28139 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03" is_pre_election: true
14:30:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.579701 28139 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 0.
14:30:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.579593 27935 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd; no voters: 
14:30:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.580439 28228 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Leader pre-election won for term 1
14:30:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.580731 28228 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:30:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.581024 28228 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 0 FOLLOWER]: Advancing to term 1
14:30:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.585774 28228 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.587622 28228 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 1 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.588655 28139 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.588603 27859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.589119 28139 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 0 FOLLOWER]: Advancing to term 1
14:30:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.589190 27859 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 0 FOLLOWER]: Advancing to term 1
14:30:18.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.593566 28139 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 1.
14:30:18.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.594533 27859 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 1.
14:30:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.594597 27933 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03; no voters: 
14:30:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.595273 28228 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 1 FOLLOWER]: Leader election won for term 1
14:30:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.597138 28228 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 1 LEADER]: Becoming Leader. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Running, Role: LEADER
14:30:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.598030 28228 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.611894 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd reported cstate change: term changed from 0 to 1, leader changed from <none> to b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194). New cstate: current_term: 1 leader_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: UNKNOWN } } }
14:30:18.686 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table a58e526f48e84e2bab5138c66df1299c
14:30:18.706 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:18.712 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299995, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:18.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [10ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=null) after lookup
14:30:18.723 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:18.725 [DEBUG - kudu-nio-3] (Connection.java:757) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] Successfully connected to peer
14:30:18.826 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x80bd4495, L:/127.0.0.1:51696 - R:/127.26.209.194:44581] using TOKEN/null
14:30:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.843602 27859 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 1 FOLLOWER]: Refusing update from remote peer b4522c04ae7349348eaf3cf1d3fd07bd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:30:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.843678 28139 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 1 FOLLOWER]: Refusing update from remote peer b4522c04ae7349348eaf3cf1d3fd07bd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:30:18.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.845714 28232 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:30:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.846557 28228 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:30:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:18.884281 28237 mvcc.cc:204] Tried to move back new op lower bound from 7305302093162426368 to 7305302092185354240. Current Snapshot: MvccSnapshot[applied={T|T < 7305302093162426368}]
14:30:19.053 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:19.055 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.194:44581
14:30:19.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:19.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:19.057351 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 27914
14:30:19.100 [DEBUG - kudu-nio-3] (Connection.java:239) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] handling channelInactive
14:30:19.102 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state READY due to: connection closed
14:30:19.104 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:19.105 [DEBUG - Time-limited test] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:19.111 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.111 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.112 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:19.127 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:19.128 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [7ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [7ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@975482337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@178ff29d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@132518b9 -> wakeup thread Time-limited test)) after lookup
14:30:19.129 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:19.137 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.137 [DEBUG - kudu-nio-5] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.137 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.160 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:19.161 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:19.162 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [7ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [7ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [23ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [32ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [32ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [55ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@975482337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@178ff29d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@132518b9 -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963))) after lookup
14:30:19.162 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:19.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.171 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.203 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:19.203 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:19.205 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=99), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [7ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [7ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [23ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [32ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [32ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [55ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [57ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [66ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [66ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [91ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [97ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@975482337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@178ff29d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@132518b9 -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675))) after lookup
14:30:19.206 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:19.209 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.210 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.210 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.222 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:19.223 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:19.223 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=118), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [7ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [7ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [23ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [32ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [32ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [55ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [57ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [66ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [66ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [91ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [97ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [101ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [105ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [105ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [116ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@975482337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@178ff29d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675) -> (continuation of Deferred@334670501 after retry RPC@1106642836), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@132518b9 -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675) -> (continuation of Deferred@334670501 after retry RPC@1106642836))) after lookup
14:30:19.225 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:19.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.228 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.462 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:19.462 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:19.463 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=358), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [7ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [7ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [23ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [32ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [32ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [55ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [57ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [66ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [66ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [91ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [97ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [101ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [105ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [105ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [116ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [119ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [122ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [123ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [357ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@975482337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@178ff29d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675) -> (continuation of Deferred@334670501 after retry RPC@1106642836) -> (continuation of Deferred@1596766023 after retry RPC@259751284), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@132518b9 -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675) -> (continuation of Deferred@334670501 after retry RPC@1106642836) -> (continuation of Deferred@1596766023 after retry RPC@259751284))) after lookup
14:30:19.464 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:19.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:19.467 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.489692 28252 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:20.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.490809 28252 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:20.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.500942 28252 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.505306 28253 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.506527 28253 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:20.513485 28074 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.513616 28253 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:20.518517 27794 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.522431 27859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a" is_pre_election: true
14:30:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.523114 27859 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 1.
14:30:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:20.523818 28074 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.525230 28074 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.526345 28252 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:30:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.526768 28252 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 1 FOLLOWER]: Starting leader election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.527132 28252 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 1 FOLLOWER]: Advancing to term 2
14:30:20.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:20.529453 27794 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.533437 28139 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03" is_pre_election: true
14:30:20.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.534101 28252 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.535293 28139 raft_consensus.cc:2393] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 71108c6bdadf491b98e0d0b98c56682a in current term 2: Already voted for candidate cdc191ea1f6b4308a5a56e280893ec03 in this term.
14:30:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.536006 28252 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 2 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:20.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.537142 27859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:20.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.537705 27859 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 1 FOLLOWER]: Advancing to term 2
14:30:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.540313 27792 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a; no voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03
14:30:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:20.540702 28074 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.543197 27859 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 2.
14:30:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.543637 28253 raft_consensus.cc:2749] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:30:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.544282 28074 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.545115 28252 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Leader election won for term 2
14:30:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.547158 28252 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 LEADER]: Becoming Leader. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Running, Role: LEADER
14:30:20.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.548214 28252 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:20.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.558610 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 reported cstate change: term changed from 1 to 2, leader changed from b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194) to cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195). New cstate: current_term: 2 leader_uuid: "cdc191ea1f6b4308a5a56e280893ec03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: HEALTHY } } }
14:30:20.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:20.842 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:20.843 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1738), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [7ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [7ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [23ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [32ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [32ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [55ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [57ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [66ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [66ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [91ms] refreshing cache from master, [92ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [97ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [101ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [105ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [105ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [116ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [119ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [122ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [123ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [351ms] refreshing cache from master, [351ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [357ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [359ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [362ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [362ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [1731ms] refreshing cache from master, [1731ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [1736ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@975482337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@178ff29d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675) -> (continuation of Deferred@334670501 after retry RPC@1106642836) -> (continuation of Deferred@1596766023 after retry RPC@259751284) -> (continuation of Deferred@524790207 after retry RPC@1266939902), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@132518b9 -> wakeup thread Time-limited test -> (continuation of Deferred@1224325656 after retry RPC@1737686963) -> (continuation of Deferred@1647650905 after retry RPC@145225675) -> (continuation of Deferred@334670501 after retry RPC@1106642836) -> (continuation of Deferred@1596766023 after retry RPC@259751284) -> (continuation of Deferred@524790207 after retry RPC@1266939902))) after lookup
14:30:20.844 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:20.845 [DEBUG - kudu-nio-2] (Connection.java:757) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] Successfully connected to peer
14:30:20.871 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xb8a7f2a9, L:/127.0.0.1:36980 - R:/127.26.209.195:44447] using TOKEN/null
14:30:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.883144 27859 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Refusing update from remote peer cdc191ea1f6b4308a5a56e280893ec03: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
14:30:20.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.884917 28252 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:30:20.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:20.887079 28074 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 -> Peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Couldn't send request to peer b4522c04ae7349348eaf3cf1d3fd07bd. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:20.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.900377 28274 mvcc.cc:204] Tried to move back new op lower bound from 7305302101520097280 to 7305302100175040512. Current Snapshot: MvccSnapshot[applied={T|T < 7305302101520097280}]
14:30:20.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.902143 28275 mvcc.cc:204] Tried to move back new op lower bound from 7305302101520097280 to 7305302100175040512. Current Snapshot: MvccSnapshot[applied={T|T < 7305302101520097280}]
14:30:20.958 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.194:44581
14:30:20.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:20.961796 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:21.306322 28281 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:21.306921 28281 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:21.307554 28281 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:21.347515 28281 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:21.348869 28281 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.194
14:30:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:21.398234 28281 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:21.399901 28281 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:21.401816 28281 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:21.418733 28290 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:22.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:21.419822 28291 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:22.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:22.703756 28293 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:22.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:22.706486 28292 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1281 milliseconds
14:30:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:22.707103 28281 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.288s	user 0.498s	sys 0.733s
14:30:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:22.707343 28281 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.289s	user 0.498s	sys 0.733s
14:30:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.707531 28281 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:22.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.708581 28281 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.711371 28281 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.712862 28281 hybrid_clock.cc:648] HybridClock initialized: now 1783521022712809 us; error 50 us; skew 500 ppm
14:30:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.713676 28281 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.720860 28281 webserver.cc:533] Webserver started at http://127.26.209.194:35263/ using document root <none> and password file <none>
14:30:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.721737 28281 fs_manager.cc:362] Metadata directory not provided
14:30:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.721937 28281 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.734505 28281 fs_manager.cc:714] Time spent opening directory manager: real 0.010s	user 0.007s	sys 0.001s
14:30:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.740895 28306 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.742218 28281 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.742683 28281 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:14 on dist-test-slave-rt66"
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.745095 28281 fs_report.cc:389] FS layout report
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/data
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:22.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.809545 28281 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:22.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.811142 28281 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:22.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.811612 28281 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.814428 28281 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:22.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.820057 28313 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.828450 28281 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.828713 28281 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
14:30:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.829046 28281 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:22.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.833853 28281 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:22.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.834059 28281 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
14:30:22.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.834456 28313 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap starting.
14:30:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.888108 28313 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Log is configured to *not* fsync() on all Append() calls
14:30:22.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.922456 28313 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.923339 28313 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap complete.
14:30:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.925175 28313 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent bootstrapping tablet: real 0.091s	user 0.065s	sys 0.024s
14:30:22.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.948695 28313 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:22.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.952109 28313 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Initialized, Role: FOLLOWER
14:30:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.953341 28313 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:22.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:22.960566 28313 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent starting tablet: real 0.035s	user 0.033s	sys 0.000s
14:30:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.003271 28281 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.194:44581
14:30:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.003509 28421 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.194:44581 every 8 connection(s)
14:30:23.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.005985 28281 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.012542 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 28281
14:30:23.013 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.049105 28422 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.049558 28422 heartbeater.cc:461] Registering TS with master...
14:30:23.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.050864 28422 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:23.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.053236 28423 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:23.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.053581 28423 heartbeater.cc:461] Registering TS with master...
14:30:23.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.054441 28423 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:23.058 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:23.059 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.195:44447
14:30:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.055866 27664 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.058929 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.194:57167
14:30:23.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:30:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.060623 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 28054
14:30:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.064410 28376 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 1 FOLLOWER]: Advancing to term 2
14:30:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.067432 27583 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.072379 28424 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.072814 28424 heartbeater.cc:461] Registering TS with master...
14:30:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.073498 28424 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:23.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.074393 28422 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:23.097620 27509 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:23.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:23.098654 28307 connection.cc:573] server connection from 127.26.209.195:49043 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:30:23.099 [DEBUG - kudu-nio-2] (Connection.java:239) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] handling channelInactive
14:30:23.099 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state READY due to: connection closed
14:30:23.103 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:23.104 [DEBUG - Time-limited test] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:23.106 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.106 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.106 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:23.121 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:23.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [16ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1862042959(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a14b369 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58ab512 -> wakeup thread Time-limited test)) after lookup
14:30:23.122 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:23.123 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.123 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.123 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.140 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:23.140 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:23.141 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [16ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [18ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [19ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [19ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1862042959(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a14b369 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58ab512 -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763))) after lookup
14:30:23.141 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:23.143 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.143 [DEBUG - kudu-nio-5] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.143 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:23.181 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:23.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [16ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [18ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [19ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [19ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [37ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [39ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [39ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [76ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1862042959(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a14b369 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58ab512 -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602))) after lookup
14:30:23.183 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:23.184 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.184 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.184 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.321 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:23.321 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:23.322 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=217), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [16ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [18ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [19ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [19ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [37ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [39ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [39ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [76ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [78ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [80ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [80ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [212ms] refreshing cache from master, [212ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [216ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1862042959(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a14b369 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602) -> (continuation of Deferred@712613720 after retry RPC@302103316), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58ab512 -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602) -> (continuation of Deferred@712613720 after retry RPC@302103316))) after lookup
14:30:23.323 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:23.325 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.325 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.325 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:23.802 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:23.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=698), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [16ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [18ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [19ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [19ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [37ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [39ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [39ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [76ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [78ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [80ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [80ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [212ms] refreshing cache from master, [212ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [216ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [218ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [221ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [221ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [692ms] refreshing cache from master, [693ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [697ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1862042959(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a14b369 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602) -> (continuation of Deferred@712613720 after retry RPC@302103316) -> (continuation of Deferred@373322787 after retry RPC@998186905), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58ab512 -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602) -> (continuation of Deferred@712613720 after retry RPC@302103316) -> (continuation of Deferred@373322787 after retry RPC@998186905))) after lookup
14:30:23.803 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:23.804 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.804 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:23.805 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.094255 28440 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Starting pre-election (detected failure of leader cdc191ea1f6b4308a5a56e280893ec03)
14:30:24.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.094913 28440 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.097117 28440 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.109045 28376 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" is_pre_election: true
14:30:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:24.110781 27792 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.111858 27794 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a; no voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03
14:30:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.112960 28440 raft_consensus.cc:2749] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
14:30:24.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.575717 28446 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 2 FOLLOWER]: Starting pre-election (detected failure of leader cdc191ea1f6b4308a5a56e280893ec03)
14:30:24.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.576260 28446 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.578775 28446 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:24.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:24.582720 28307 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:24.587610 28307 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:24.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.591876 27859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a" is_pre_election: true
14:30:24.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.592474 27859 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 2.
14:30:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.593624 28309 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd; no voters: cdc191ea1f6b4308a5a56e280893ec03
14:30:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.594273 28446 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 2 FOLLOWER]: Leader pre-election won for term 3
14:30:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.594578 28446 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 2 FOLLOWER]: Starting leader election (detected failure of leader cdc191ea1f6b4308a5a56e280893ec03)
14:30:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.594807 28446 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 2 FOLLOWER]: Advancing to term 3
14:30:24.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.599148 28446 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:24.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.600643 28446 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 3 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.601687 27859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.602247 27859 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 2 FOLLOWER]: Advancing to term 3
14:30:24.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:24.604658 28307 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:24.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.606985 27859 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 3.
14:30:24.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.607935 28309 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd; no voters: cdc191ea1f6b4308a5a56e280893ec03
14:30:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.608639 28446 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 3 FOLLOWER]: Leader election won for term 3
14:30:24.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.609748 28446 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 3 LEADER]: Becoming Leader. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Running, Role: LEADER
14:30:24.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.611399 28446 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:24.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.621644 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd reported cstate change: term changed from 2 to 3, leader changed from cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195) to b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194). New cstate: current_term: 3 leader_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: UNKNOWN } } }
14:30:24.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:24.802 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:24.803 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1698), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [16ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [18ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [19ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [19ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [37ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [39ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [39ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [76ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [78ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [80ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [80ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [212ms] refreshing cache from master, [212ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [216ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [218ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [221ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [221ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [692ms] refreshing cache from master, [693ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [697ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [699ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [700ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [701ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [1692ms] refreshing cache from master, [1692ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [1697ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1862042959(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a14b369 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602) -> (continuation of Deferred@712613720 after retry RPC@302103316) -> (continuation of Deferred@373322787 after retry RPC@998186905) -> (continuation of Deferred@880054275 after retry RPC@1398550464), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58ab512 -> wakeup thread Time-limited test -> (continuation of Deferred@2051082620 after retry RPC@707989763) -> (continuation of Deferred@872734379 after retry RPC@1196421602) -> (continuation of Deferred@712613720 after retry RPC@302103316) -> (continuation of Deferred@373322787 after retry RPC@998186905) -> (continuation of Deferred@880054275 after retry RPC@1398550464))) after lookup
14:30:24.803 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:24.804 [DEBUG - kudu-nio-1] (Connection.java:757) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] Successfully connected to peer
14:30:24.832 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xb59fd9ef, L:/127.0.0.1:36644 - R:/127.26.209.194:44581] using TOKEN/null
14:30:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.842803 27859 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 3 FOLLOWER]: Refusing update from remote peer b4522c04ae7349348eaf3cf1d3fd07bd: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
14:30:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.844331 28446 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
14:30:24.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:24.845628 28307 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd -> Peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Couldn't send request to peer cdc191ea1f6b4308a5a56e280893ec03. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:24.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.856463 28464 mvcc.cc:204] Tried to move back new op lower bound from 7305302117740204032 to 7305302116816068608. Current Snapshot: MvccSnapshot[applied={T|T < 7305302117740204032}]
14:30:24.901 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.195:44447
14:30:24.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:24.904243 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:44447
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39951
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:25.217810 28471 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:25.218422 28471 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:25.219055 28471 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:25.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:25.253049 28471 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:25.254071 28471 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.195
14:30:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:25.291463 28471 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:44447
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39951
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:25.293540 28471 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:25.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:25.295233 28471 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:25.307905 28477 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:25.309440 28478 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:26.678750 28480 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:26.680631 28479 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1366 milliseconds
14:30:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:26.681538 28471 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.372s	user 0.464s	sys 0.845s
14:30:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:26.681805 28471 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.373s	user 0.464s	sys 0.845s
14:30:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.682106 28471 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:26.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.683256 28471 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:26.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.685527 28471 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.686890 28471 hybrid_clock.cc:648] HybridClock initialized: now 1783521026686842 us; error 58 us; skew 500 ppm
14:30:26.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.687791 28471 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.694928 28471 webserver.cc:533] Webserver started at http://127.26.209.195:39951/ using document root <none> and password file <none>
14:30:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.695976 28471 fs_manager.cc:362] Metadata directory not provided
14:30:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.696440 28471 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.704607 28471 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.001s
14:30:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.709925 28490 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:26.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.711115 28471 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
14:30:26.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.711467 28471 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:26.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:26.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:16 on dist-test-slave-rt66"
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.713537 28471 fs_report.cc:389] FS layout report
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/data
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:26.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.796221 28471 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:26.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.797807 28471 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:26.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.798287 28471 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:26.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.800899 28471 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:26.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.808620 28497 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:26.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.820123 28471 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:26.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.820436 28471 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.014s	user 0.001s	sys 0.001s
14:30:26.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.820766 28471 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.827252 28471 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.827526 28471 ts_tablet_manager.cc:595] Time spent register tablets: real 0.007s	user 0.005s	sys 0.000s
14:30:26.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.828015 28497 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap starting.
14:30:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.885043 28497 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Log is configured to *not* fsync() on all Append() calls
14:30:26.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.935657 28497 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.936749 28497 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap complete.
14:30:26.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.938159 28497 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent bootstrapping tablet: real 0.111s	user 0.050s	sys 0.053s
14:30:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.956847 28497 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.959954 28497 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Initialized, Role: FOLLOWER
14:30:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.961040 28497 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:26.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.965144 28497 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent starting tablet: real 0.027s	user 0.025s	sys 0.003s
14:30:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.994457 28471 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.195:44447
14:30:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.994673 28605 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.195:44447 every 8 connection(s)
14:30:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:26.997210 28471 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:27.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.006835 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 28471
14:30:27.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:27.054 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:27.056 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.194:44581
14:30:27.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.056340 28608 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.056901 28608 heartbeater.cc:461] Registering TS with master...
14:30:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.057240 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 28281
14:30:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.058517 28608 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.063575 27509 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.067008 28607 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.067375 28607 heartbeater.cc:461] Registering TS with master...
14:30:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.068186 28607 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.071826 27583 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:27.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.074172 28606 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:27.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.074514 28606 heartbeater.cc:461] Registering TS with master...
14:30:27.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.075376 28606 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:27.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.078145 27664 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:27.080958 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.195:58631
14:30:27.091 [DEBUG - kudu-nio-1] (Connection.java:239) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] handling channelInactive
14:30:27.091 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:27.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state READY due to: connection closed
14:30:27.092 [DEBUG - Time-limited test] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:27.093 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.094 [DEBUG - kudu-nio-2] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.094 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.120 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:27.120 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:27.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test)) after lookup
14:30:27.121 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:27.122 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.122 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.122 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.140 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:27.141 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:27.141 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [29ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [30ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [48ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995))) after lookup
14:30:27.142 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:27.143 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.143 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.143 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:27.180 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:27.180 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [29ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [30ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [48ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [49ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [51ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [51ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030))) after lookup
14:30:27.181 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:27.182 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.182 [DEBUG - kudu-nio-5] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.182 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.200 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:27.201 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:27.201 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=109), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [29ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [30ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [48ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [49ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [51ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [51ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [90ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [108ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360))) after lookup
14:30:27.202 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:27.203 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.203 [DEBUG - kudu-nio-6] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.203 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:27.441 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:27.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=349), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [29ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [30ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [48ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [49ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [51ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [51ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [90ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [108ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [109ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [111ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [111ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [344ms] refreshing cache from master, [344ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [348ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769))) after lookup
14:30:27.442 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:27.443 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.443 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.443 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:27.801 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:27.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=710), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [29ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [30ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [48ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [49ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [51ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [51ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [90ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [108ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [109ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [111ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [111ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [344ms] refreshing cache from master, [344ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [348ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [350ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [351ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [351ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [703ms] refreshing cache from master, [704ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [708ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769) -> (continuation of Deferred@1072948408 after retry RPC@1406882844), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769) -> (continuation of Deferred@1072948408 after retry RPC@1406882844))) after lookup
14:30:27.802 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:27.803 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.803 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:27.804 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:28.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:28.022 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:28.023 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=931), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [29ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [30ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [48ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [49ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [51ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [51ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [90ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [108ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [109ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [111ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [111ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [344ms] refreshing cache from master, [344ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [348ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [350ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [351ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [351ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [703ms] refreshing cache from master, [704ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [708ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [710ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [711ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [712ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [924ms] refreshing cache from master, [924ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [930ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769) -> (continuation of Deferred@1072948408 after retry RPC@1406882844) -> (continuation of Deferred@969238835 after retry RPC@860720070), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769) -> (continuation of Deferred@1072948408 after retry RPC@1406882844) -> (continuation of Deferred@969238835 after retry RPC@860720070))) after lookup
14:30:28.024 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:28.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:28.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:28.026 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.084558 28606 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.251360 28618 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:30:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.252056 28618 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:28.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.255110 28618 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:28.261106 28493 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:28.265610 28493 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:28.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.267130 27859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a" is_pre_election: true
14:30:28.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.268685 28493 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd
14:30:28.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.269433 28618 raft_consensus.cc:2749] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
14:30:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.348114 28624 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 3 FOLLOWER]: Starting pre-election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.348637 28624 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.350543 28624 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:28.354578 27794 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:30:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:28.359679 27794 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.367092 28560 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03" is_pre_election: true
14:30:28.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.367945 28560 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 71108c6bdadf491b98e0d0b98c56682a in term 2.
14:30:28.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.369589 27792 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:28.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.370342 28624 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 3 FOLLOWER]: Leader pre-election won for term 4
14:30:28.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.370684 28624 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 3 FOLLOWER]: Starting leader election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:28.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.370932 28624 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 3 FOLLOWER]: Advancing to term 4
14:30:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.376367 28624 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:28.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.378203 28624 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 4 election: Requested vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.379618 28560 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.380213 28560 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 2 FOLLOWER]: Advancing to term 4
14:30:28.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:28.382222 27794 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.387916 28560 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 71108c6bdadf491b98e0d0b98c56682a in term 4.
14:30:28.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.389107 27792 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:28.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.389775 28624 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 4 FOLLOWER]: Leader election won for term 4
14:30:28.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.391070 28624 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 4 LEADER]: Becoming Leader. State: Replica: 71108c6bdadf491b98e0d0b98c56682a, State: Running, Role: LEADER
14:30:28.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.392169 28624 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.402539 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a reported cstate change: term changed from 3 to 4, leader changed from b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194) to 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193). New cstate: current_term: 4 leader_uuid: "71108c6bdadf491b98e0d0b98c56682a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: UNKNOWN } } }
14:30:28.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.902284 28560 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 4 FOLLOWER]: Refusing update from remote peer 71108c6bdadf491b98e0d0b98c56682a: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
14:30:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.904083 28642 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
14:30:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:28.919011 28644 mvcc.cc:204] Tried to move back new op lower bound from 7305302117740204032 to 7305302116816068608. Current Snapshot: MvccSnapshot[applied={T|T < 7305302117740204032}]
14:30:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:28.956616 27794 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a -> Peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Couldn't send request to peer b4522c04ae7349348eaf3cf1d3fd07bd. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:30.861 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:30.862 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:30.862 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=16, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3770), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [28ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [29ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [30ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [48ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [49ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [51ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [51ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [90ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [108ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [109ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [111ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [111ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [344ms] refreshing cache from master, [344ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [348ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [350ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [351ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [351ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [703ms] refreshing cache from master, [704ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [708ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [710ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [711ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [712ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [924ms] refreshing cache from master, [924ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [930ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [931ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [933ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [934ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [3764ms] refreshing cache from master, [3764ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [3769ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@35406349(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@300c4fa1 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769) -> (continuation of Deferred@1072948408 after retry RPC@1406882844) -> (continuation of Deferred@969238835 after retry RPC@860720070) -> (continuation of Deferred@1043387708 after retry RPC@1635547783), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ff7bad6 -> wakeup thread Time-limited test -> (continuation of Deferred@511038297 after retry RPC@1545009995) -> (continuation of Deferred@1840896497 after retry RPC@111066030) -> (continuation of Deferred@1034041129 after retry RPC@1318234360) -> (continuation of Deferred@1586728299 after retry RPC@85801769) -> (continuation of Deferred@1072948408 after retry RPC@1406882844) -> (continuation of Deferred@969238835 after retry RPC@860720070) -> (continuation of Deferred@1043387708 after retry RPC@1635547783))) after lookup
14:30:30.863 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:30.864 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] Successfully connected to peer
14:30:30.893 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x79b1b846, L:/127.0.0.1:39396 - R:/127.26.209.193:36347] using TOKEN/null
14:30:30.946 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.194:44581
14:30:30.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:30.948601 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.305768 28670 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.306411 28670 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.306944 28670 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.342628 28670 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.343706 28670 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.194
14:30:31.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.362318 27794 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a -> Peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Couldn't send request to peer b4522c04ae7349348eaf3cf1d3fd07bd. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:30:31.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:31.390214 28670 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:31.392339 28670 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:31.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:31.394584 28670 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.411541 28677 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:32.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.412684 28678 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:32.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:31.416715 28680 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:32.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:32.591485 28679 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1175 milliseconds
14:30:32.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.591629 28670 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:32.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.593014 28670 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:32.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.595306 28670 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:32.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.596696 28670 hybrid_clock.cc:648] HybridClock initialized: now 1783521032596661 us; error 53 us; skew 500 ppm
14:30:32.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.597649 28670 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:32.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.604750 28670 webserver.cc:533] Webserver started at http://127.26.209.194:35263/ using document root <none> and password file <none>
14:30:32.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.606022 28670 fs_manager.cc:362] Metadata directory not provided
14:30:32.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.606247 28670 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:32.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.614315 28670 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
14:30:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.619022 28692 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:32.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.620114 28670 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
14:30:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.620471 28670 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:14 on dist-test-slave-rt66"
14:30:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.622435 28670 fs_report.cc:389] FS layout report
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/data
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:32.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.666828 28670 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.668429 28670 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:32.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.668964 28670 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:32.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.671722 28670 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.677702 28699 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.686538 28670 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.686851 28670 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
14:30:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.687175 28670 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.692162 28670 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.692355 28670 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.000s	sys 0.002s
14:30:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.692842 28699 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap starting.
14:30:32.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.758612 28699 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Log is configured to *not* fsync() on all Append() calls
14:30:32.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.812697 28699 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:32.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.813771 28699 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap complete.
14:30:32.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.814985 28699 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent bootstrapping tablet: real 0.123s	user 0.084s	sys 0.034s
14:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.835223 28699 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.838048 28699 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Initialized, Role: FOLLOWER
14:30:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.839300 28699 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:32.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.843986 28699 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent starting tablet: real 0.029s	user 0.017s	sys 0.012s
14:30:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.861552 28670 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.194:44581
14:30:32.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.861886 28807 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.194:44581 every 8 connection(s)
14:30:32.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.864202 28670 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:32.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.873059 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 28670
14:30:32.873 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:32.891 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:32.913 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.193:36347
14:30:32.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:30:32.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.914202 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 27749
14:30:32.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.916571 28810 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:32.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.917240 28810 heartbeater.cc:461] Registering TS with master...
14:30:32.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.918704 28810 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:32.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.920276 28808 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:32.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.920648 28808 heartbeater.cc:461] Registering TS with master...
14:30:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.921608 28808 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:32.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.923599 27509 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:32.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.924433 27664 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.927171 28809 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.927306 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.194:33527
14:30:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.927522 28809 heartbeater.cc:461] Registering TS with master...
14:30:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.928349 28809 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:32.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:32.931630 27583 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:32.946 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] handling channelInactive
14:30:32.947 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state READY due to: connection closed
14:30:32.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:32.948 [DEBUG - Time-limited test] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:32.949 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.949 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.950 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:32.959 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:32.960 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [1ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [11ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@836660740(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@33f492fa -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7815474d -> wakeup thread Time-limited test)) after lookup
14:30:32.961 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:32.963 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.963 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.963 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.979 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:32.980 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:32.980 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [1ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [11ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [12ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [15ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [31ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@836660740(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@33f492fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7815474d -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290))) after lookup
14:30:32.980 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:32.981 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.982 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.982 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:32.999 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:33.000 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:33.000 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [1ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [11ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [12ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [15ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [31ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [32ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [34ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [34ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [51ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@836660740(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@33f492fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7815474d -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785))) after lookup
14:30:33.001 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:33.002 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.002 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.002 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.059 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:33.059 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:33.060 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=112), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [1ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [11ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [12ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [15ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [31ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [32ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [34ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [34ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [51ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [52ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [54ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [54ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [107ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [111ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@836660740(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@33f492fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785) -> (continuation of Deferred@1302810330 after retry RPC@1880766255), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7815474d -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785) -> (continuation of Deferred@1302810330 after retry RPC@1880766255))) after lookup
14:30:33.060 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:33.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.062 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.481 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:33.481 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:33.482 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=534), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [1ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [11ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [12ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [15ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [31ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [32ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [34ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [34ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [51ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [52ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [54ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [54ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [107ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [111ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [112ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [114ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [114ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [528ms] refreshing cache from master, [528ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [532ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@836660740(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@33f492fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785) -> (continuation of Deferred@1302810330 after retry RPC@1880766255) -> (continuation of Deferred@1479234513 after retry RPC@2041164980), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7815474d -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785) -> (continuation of Deferred@1302810330 after retry RPC@1880766255) -> (continuation of Deferred@1479234513 after retry RPC@2041164980))) after lookup
14:30:33.482 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:33.483 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.484 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.484 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:33.932519 28808 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.087105 28820 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 71108c6bdadf491b98e0d0b98c56682a)
14:30:34.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.087677 28820 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:34.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.089572 28820 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:34.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:34.092418 28493 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:34.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:34.096905 28493 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.103528 28762 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" is_pre_election: true
14:30:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.104247 28762 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 3.
14:30:34.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.105541 28493 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a
14:30:34.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.106307 28820 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 4 FOLLOWER]: Leader pre-election won for term 5
14:30:34.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.106630 28820 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 71108c6bdadf491b98e0d0b98c56682a)
14:30:34.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.106947 28820 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 4 FOLLOWER]: Advancing to term 5
14:30:34.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.112026 28820 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.113755 28820 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 5 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.115662 28762 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.116129 28762 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 3 FOLLOWER]: Advancing to term 5
14:30:34.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:34.117663 28493 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.122826 28762 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 5.
14:30:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.123886 28493 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a
14:30:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.124706 28820 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 5 FOLLOWER]: Leader election won for term 5
14:30:34.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.125854 28820 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 5 LEADER]: Becoming Leader. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Running, Role: LEADER
14:30:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.126772 28820 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:34.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.134630 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 reported cstate change: term changed from 4 to 5, leader changed from 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193) to cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195). New cstate: current_term: 5 leader_uuid: "cdc191ea1f6b4308a5a56e280893ec03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: HEALTHY } } }
14:30:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:34.576046 28493 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 -> Peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Couldn't send request to peer 71108c6bdadf491b98e0d0b98c56682a. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.670073 28762 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 5 FOLLOWER]: Refusing update from remote peer cdc191ea1f6b4308a5a56e280893ec03: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
14:30:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:34.671700 28820 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
14:30:35.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:35.121 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:35.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2174), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [1ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [11ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [12ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [15ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [31ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [32ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [34ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [34ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [47ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [51ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [52ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [54ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [54ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [107ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [111ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [112ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [114ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [114ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [528ms] refreshing cache from master, [528ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [532ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [534ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [536ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [536ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2168ms] refreshing cache from master, [2168ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [2173ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@836660740(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@33f492fa -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785) -> (continuation of Deferred@1302810330 after retry RPC@1880766255) -> (continuation of Deferred@1479234513 after retry RPC@2041164980) -> (continuation of Deferred@1963213646 after retry RPC@1194589891), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7815474d -> wakeup thread Time-limited test -> (continuation of Deferred@765815475 after retry RPC@380005290) -> (continuation of Deferred@396311123 after retry RPC@851572785) -> (continuation of Deferred@1302810330 after retry RPC@1880766255) -> (continuation of Deferred@1479234513 after retry RPC@2041164980) -> (continuation of Deferred@1963213646 after retry RPC@1194589891))) after lookup
14:30:35.123 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:35.124 [DEBUG - kudu-nio-1] (Connection.java:757) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] Successfully connected to peer
14:30:35.145 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xc9ec9d57, L:/127.0.0.1:59092 - R:/127.26.209.195:44447] using TOKEN/null
14:30:35.197 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.193:36347
14:30:35.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:30:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:35.202016 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:30:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:36347
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45851
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:35.536420 28852 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:35.536944 28852 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:35.537417 28852 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:35.571337 28852 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:35.572232 28852 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.193
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:35.614601 28852 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:36347
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45851
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:35.615959 28852 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:35.617666 28852 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:35.633119 28858 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:36.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:35.633360 28859 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:36.977339 28852 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.345s	user 0.445s	sys 0.819s
14:30:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:36.977710 28852 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.345s	user 0.445s	sys 0.819s
14:30:36.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:36.978753 28860 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1345 milliseconds
14:30:36.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:36.979229 28861 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:36.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:36.980886 28852 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:36.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:36.982311 28852 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:36.984970 28852 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:36.986407 28852 hybrid_clock.cc:648] HybridClock initialized: now 1783521036986420 us; error 127 us; skew 500 ppm
14:30:36.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:36.987787 28852 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:37.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.007434 28852 webserver.cc:533] Webserver started at http://127.26.209.193:45851/ using document root <none> and password file <none>
14:30:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.008721 28852 fs_manager.cc:362] Metadata directory not provided
14:30:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.009063 28852 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.019441 28852 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.000s	sys 0.004s
14:30:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.025646 28873 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:37.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.026901 28852 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
14:30:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.027201 28852 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:13 on dist-test-slave-rt66"
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.029491 28852 fs_report.cc:389] FS layout report
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/data
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:37.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.123265 28852 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:37.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.124815 28852 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:37.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.125231 28852 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:37.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.127815 28852 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.133966 28880 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.142246 28852 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.142511 28852 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
14:30:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.142728 28852 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.147316 28852 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.147572 28852 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
14:30:37.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.147958 28880 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap starting.
14:30:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.222920 28880 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Log is configured to *not* fsync() on all Append() calls
14:30:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:37.225803 28493 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 -> Peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Couldn't send request to peer 71108c6bdadf491b98e0d0b98c56682a. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:30:37.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.281822 28880 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:37.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.282927 28880 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap complete.
14:30:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.284578 28880 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent bootstrapping tablet: real 0.137s	user 0.088s	sys 0.043s
14:30:37.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.302789 28880 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:37.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.305505 28880 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 71108c6bdadf491b98e0d0b98c56682a, State: Initialized, Role: FOLLOWER
14:30:37.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.306671 28880 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.310762 28880 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent starting tablet: real 0.026s	user 0.024s	sys 0.000s
14:30:37.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.320683 28852 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.193:36347
14:30:37.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.320955 28988 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.193:36347 every 8 connection(s)
14:30:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.323256 28852 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:37.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.325524 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 28852
14:30:37.366 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:37.366 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.195:44447
14:30:37.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:30:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.367686 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 28471
14:30:37.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.361212 28989 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:37.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.368868 28990 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:37.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.370666 28989 heartbeater.cc:461] Registering TS with master...
14:30:37.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.370867 28990 heartbeater.cc:461] Registering TS with master...
14:30:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.372129 28989 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.372179 28990 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:37.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.376303 27664 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.378683 27584 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.379027 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.193:52615
14:30:37.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.379580 28991 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:37.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.379964 28991 heartbeater.cc:461] Registering TS with master...
14:30:37.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.380986 28991 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:37.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:37.384718 27509 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:37.397 [DEBUG - kudu-nio-1] (Connection.java:239) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] handling channelInactive
14:30:37.397 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state READY due to: connection closed
14:30:37.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:37.398 [DEBUG - Time-limited test] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:37.399 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.400 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.400 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.419 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:37.419 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:37.420 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1682753975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3cd1f61b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4f56db2a -> wakeup thread Time-limited test)) after lookup
14:30:37.420 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:37.421 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.421 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.421 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:37.441 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:37.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [22ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [23ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [42ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1682753975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3cd1f61b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4f56db2a -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627))) after lookup
14:30:37.442 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:37.443 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.443 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.443 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.480 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:37.480 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:37.487 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [22ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [23ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [42ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [43ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [45ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [81ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1682753975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3cd1f61b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4f56db2a -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180))) after lookup
14:30:37.487 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:37.491 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.491 [DEBUG - kudu-nio-5] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.491 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.600 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:37.600 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:37.600 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=202), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [22ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [23ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [42ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [43ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [45ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [81ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [93ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [93ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [202ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1682753975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3cd1f61b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180) -> (continuation of Deferred@208397455 after retry RPC@1620555716), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4f56db2a -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180) -> (continuation of Deferred@208397455 after retry RPC@1620555716))) after lookup
14:30:37.601 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:37.602 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.602 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:37.602 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:38.062 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:38.063 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:38.063 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=664), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [22ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [23ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [42ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [43ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [45ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [81ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [93ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [93ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [202ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [203ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [204ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [204ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [658ms] refreshing cache from master, [658ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [664ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1682753975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3cd1f61b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180) -> (continuation of Deferred@208397455 after retry RPC@1620555716) -> (continuation of Deferred@2123528170 after retry RPC@602330731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4f56db2a -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180) -> (continuation of Deferred@208397455 after retry RPC@1620555716) -> (continuation of Deferred@2123528170 after retry RPC@602330731))) after lookup
14:30:38.064 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:38.065 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:38.065 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:38.065 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.383735 28989 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:38.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.505064 29000 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 5 FOLLOWER]: Starting pre-election (detected failure of leader cdc191ea1f6b4308a5a56e280893ec03)
14:30:38.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.505527 29000 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.508194 29000 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:38.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:38.513414 28693 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:38.517733 28693 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.522904 28943 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a" is_pre_election: true
14:30:38.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.523767 28943 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 4.
14:30:38.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.525509 28695 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd; no voters: cdc191ea1f6b4308a5a56e280893ec03
14:30:38.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.526379 29000 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 5 FOLLOWER]: Leader pre-election won for term 6
14:30:38.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.526806 29000 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 5 FOLLOWER]: Starting leader election (detected failure of leader cdc191ea1f6b4308a5a56e280893ec03)
14:30:38.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.527115 29000 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 5 FOLLOWER]: Advancing to term 6
14:30:38.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.531823 29000 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.533432 29000 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 6 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:38.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.534673 28943 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:38.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.535367 28943 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 4 FOLLOWER]: Advancing to term 6
14:30:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:38.537896 28693 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:38.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.543562 28943 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 6.
14:30:38.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.544909 28695 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd; no voters: cdc191ea1f6b4308a5a56e280893ec03
14:30:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.545728 29000 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Leader election won for term 6
14:30:38.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.547451 29000 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 LEADER]: Becoming Leader. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Running, Role: LEADER
14:30:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.548278 29000 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.559190 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd reported cstate change: term changed from 5 to 6, leader changed from cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195) to b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194). New cstate: current_term: 6 leader_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: UNKNOWN } } }
14:30:38.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:38.880 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:38.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1482), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [17ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [21ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [22ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [23ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [42ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [43ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [45ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [45ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [81ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [89ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [93ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [93ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [202ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [203ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [204ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [204ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [658ms] refreshing cache from master, [658ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [664ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [665ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [667ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [667ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [1477ms] refreshing cache from master, [1478ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [1481ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1682753975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3cd1f61b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180) -> (continuation of Deferred@208397455 after retry RPC@1620555716) -> (continuation of Deferred@2123528170 after retry RPC@602330731) -> (continuation of Deferred@1700515878 after retry RPC@1889208168), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4f56db2a -> wakeup thread Time-limited test -> (continuation of Deferred@1644908523 after retry RPC@469583627) -> (continuation of Deferred@2100903415 after retry RPC@1862300180) -> (continuation of Deferred@208397455 after retry RPC@1620555716) -> (continuation of Deferred@2123528170 after retry RPC@602330731) -> (continuation of Deferred@1700515878 after retry RPC@1889208168))) after lookup
14:30:38.881 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:38.882 [DEBUG - kudu-nio-0] (Connection.java:757) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] Successfully connected to peer
14:30:38.907 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x28a6e0b7, L:/127.0.0.1:42766 - R:/127.26.209.194:44581] using TOKEN/null
14:30:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.919703 28943 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 6 FOLLOWER]: Refusing update from remote peer b4522c04ae7349348eaf3cf1d3fd07bd: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 22. (index mismatch)
14:30:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:38.921456 28693 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd -> Peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Couldn't send request to peer cdc191ea1f6b4308a5a56e280893ec03. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.922152 29000 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
14:30:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.939567 29020 mvcc.cc:204] Tried to move back new op lower bound from 7305302159972831232 to 7305302155791585280. Current Snapshot: MvccSnapshot[applied={T|T < 7305302159972831232}]
14:30:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.950687 29018 mvcc.cc:204] Tried to move back new op lower bound from 7305302175393284096 to 7305302173901967360. Current Snapshot: MvccSnapshot[applied={T|T < 7305302175393284096}]
14:30:38.996 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.195:44447
14:30:38.998 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:30:38.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:38.999519 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:38.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:38.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:44447
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39951
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:39.313164 29027 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:39.313637 29027 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:39.314101 29027 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:39.369545 29027 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:39.370918 29027 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.195
14:30:39.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:39.412474 29027 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:44447
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39951
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:39.413810 29027 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:39.415354 29027 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:39.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:39.427186 29034 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:40.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:39.429005 29035 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:40.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:40.638067 29037 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:40.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:40.639962 29036 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1207 milliseconds
14:30:40.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.640147 29027 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:40.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.641482 29027 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.644248 29027 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:40.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.645648 29027 hybrid_clock.cc:648] HybridClock initialized: now 1783521040645616 us; error 47 us; skew 500 ppm
14:30:40.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.646473 29027 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.652956 29027 webserver.cc:533] Webserver started at http://127.26.209.195:39951/ using document root <none> and password file <none>
14:30:40.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.654085 29027 fs_manager.cc:362] Metadata directory not provided
14:30:40.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.654363 29027 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.662062 29027 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.001s
14:30:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.666631 29046 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.667798 29027 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.668104 29027 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:16 on dist-test-slave-rt66"
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.670063 29027 fs_report.cc:389] FS layout report
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/data
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.725026 29027 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.726516 29027 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:40.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.726943 29027 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.729504 29027 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:40.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.735913 29053 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.744733 29027 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.745100 29027 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.001s
14:30:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.745424 29027 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:40.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.753003 29027 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:40.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.753289 29027 ts_tablet_manager.cc:595] Time spent register tablets: real 0.008s	user 0.001s	sys 0.007s
14:30:40.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.753855 29053 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap starting.
14:30:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.822213 29053 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Log is configured to *not* fsync() on all Append() calls
14:30:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.894990 29053 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:40.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.896056 29053 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap complete.
14:30:40.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.897971 29053 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent bootstrapping tablet: real 0.145s	user 0.096s	sys 0.043s
14:30:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.916828 29053 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.919405 29053 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Initialized, Role: FOLLOWER
14:30:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.920457 29053 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.925165 29053 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent starting tablet: real 0.027s	user 0.025s	sys 0.002s
14:30:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.943050 29027 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.195:44447
14:30:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.943432 29163 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.195:44447 every 8 connection(s)
14:30:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.945885 29027 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:40.950 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.948670 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 29027
14:30:40.973 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:40.974 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.194:44581
14:30:40.974 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:40.976428 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 28670
14:30:41.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.010854 29166 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.011102 29164 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.011610 29165 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.011843 29164 heartbeater.cc:461] Registering TS with master...
14:30:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.012243 29165 heartbeater.cc:461] Registering TS with master...
14:30:41.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.013537 29165 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:41.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.013608 29164 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:41.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.016119 29166 heartbeater.cc:461] Registering TS with master...
14:30:41.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.017306 29166 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:41.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.018421 27584 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.020344 27664 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:41.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.020680 27509 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:41.022 [DEBUG - kudu-nio-0] (Connection.java:239) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] handling channelInactive
14:30:41.023 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state READY due to: connection closed
14:30:41.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:41.026 [DEBUG - Time-limited test] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:41.023305 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.195:36661
14:30:41.027 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.027 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.027 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:41.040 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:41.041 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [15ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@968192373(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a511e1e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d229176 -> wakeup thread Time-limited test)) after lookup
14:30:41.041 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:41.042 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.042 [DEBUG - kudu-nio-2] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.042 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.060 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:41.060 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:41.061 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [15ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [16ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [17ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@968192373(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a511e1e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d229176 -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891))) after lookup
14:30:41.061 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:41.062 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.063 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.063 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.079 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:41.079 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:41.080 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [15ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [16ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [17ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [36ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [38ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [54ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@968192373(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a511e1e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d229176 -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425))) after lookup
14:30:41.080 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:41.081 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.081 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.081 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.200 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:41.200 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:41.200 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=175), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [15ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [16ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [17ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [36ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [38ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [54ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [55ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [56ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [56ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [174ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@968192373(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a511e1e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d229176 -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733))) after lookup
14:30:41.201 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:41.202 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.202 [DEBUG - kudu-nio-5] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.202 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.460 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:41.460 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:41.460 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=435), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [15ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [16ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [17ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [36ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [38ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [54ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [55ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [56ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [56ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [174ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [176ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [177ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [177ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [430ms] refreshing cache from master, [431ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [434ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@968192373(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a511e1e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733) -> (continuation of Deferred@535820292 after retry RPC@266874173), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d229176 -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733) -> (continuation of Deferred@535820292 after retry RPC@266874173))) after lookup
14:30:41.461 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:41.462 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.462 [DEBUG - kudu-nio-6] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.462 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.939 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:41.940 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:41.940 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=915), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [15ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [16ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [17ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [36ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [38ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [54ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [55ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [56ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [56ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [174ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [176ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [177ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [177ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [430ms] refreshing cache from master, [431ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [434ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [435ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [437ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [437ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [910ms] refreshing cache from master, [911ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [914ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@968192373(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a511e1e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733) -> (continuation of Deferred@535820292 after retry RPC@266874173) -> (continuation of Deferred@404947192 after retry RPC@1458325252), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d229176 -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733) -> (continuation of Deferred@535820292 after retry RPC@266874173) -> (continuation of Deferred@404947192 after retry RPC@1458325252))) after lookup
14:30:41.941 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:41.942 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.942 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:41.942 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.029626 29164 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.070510 29176 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 6 FOLLOWER]: Starting pre-election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.071076 29176 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:42.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.074019 29176 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:42.077847 28876 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:42.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:42.082024 28876 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.086984 29116 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03" is_pre_election: true
14:30:42.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.087783 29116 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 71108c6bdadf491b98e0d0b98c56682a in term 5.
14:30:42.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.089444 28874 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:42.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.090229 29176 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 6 FOLLOWER]: Leader pre-election won for term 7
14:30:42.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.090597 29176 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 6 FOLLOWER]: Starting leader election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.090890 29176 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 6 FOLLOWER]: Advancing to term 7
14:30:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.096221 29176 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.098037 29176 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 7 election: Requested vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:42.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.099697 29116 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:42.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.100364 29116 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 5 FOLLOWER]: Advancing to term 7
14:30:42.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:42.102496 28876 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:42.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.107532 29116 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 71108c6bdadf491b98e0d0b98c56682a in term 7.
14:30:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.109033 28874 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:42.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.109715 29176 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 7 FOLLOWER]: Leader election won for term 7
14:30:42.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.111349 29176 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 7 LEADER]: Becoming Leader. State: Replica: 71108c6bdadf491b98e0d0b98c56682a, State: Running, Role: LEADER
14:30:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.112210 29176 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:42.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.122179 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a reported cstate change: term changed from 6 to 7, leader changed from b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194) to 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193). New cstate: current_term: 7 leader_uuid: "71108c6bdadf491b98e0d0b98c56682a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: UNKNOWN } } }
14:30:42.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.605683 29116 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 7 FOLLOWER]: Refusing update from remote peer 71108c6bdadf491b98e0d0b98c56682a: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
14:30:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:42.607333 29176 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
14:30:42.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:42.656996 28876 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a -> Peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Couldn't send request to peer b4522c04ae7349348eaf3cf1d3fd07bd. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:43.701 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:43.701 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:43.702 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=2676), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [15ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [16ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [17ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [17ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [35ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [36ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [38ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [38ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [54ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [55ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [56ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [56ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [174ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [176ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [177ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [177ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [430ms] refreshing cache from master, [431ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [434ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [435ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [437ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [437ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [910ms] refreshing cache from master, [911ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [914ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [916ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [917ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [917ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2671ms] refreshing cache from master, [2671ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [2676ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@968192373(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@3a511e1e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733) -> (continuation of Deferred@535820292 after retry RPC@266874173) -> (continuation of Deferred@404947192 after retry RPC@1458325252) -> (continuation of Deferred@460494566 after retry RPC@798107371), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2d229176 -> wakeup thread Time-limited test -> (continuation of Deferred@1851656934 after retry RPC@998375891) -> (continuation of Deferred@411928282 after retry RPC@858423425) -> (continuation of Deferred@680675982 after retry RPC@1331427733) -> (continuation of Deferred@535820292 after retry RPC@266874173) -> (continuation of Deferred@404947192 after retry RPC@1458325252) -> (continuation of Deferred@460494566 after retry RPC@798107371))) after lookup
14:30:43.702 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:43.703 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] Successfully connected to peer
14:30:43.721 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xe1d955d1, L:/127.0.0.1:42960 - R:/127.26.209.193:36347] using TOKEN/null
14:30:43.783 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.194:44581
14:30:43.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:43.785405 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:44.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.144812 29216 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:44.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.145399 29216 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.145977 29216 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:44.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.179437 29216 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:44.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.180428 29216 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.194
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:44.216749 29216 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:44.218531 29216 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:44.220286 29216 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.233546 29223 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:44.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.802114 28876 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a -> Peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Couldn't send request to peer b4522c04ae7349348eaf3cf1d3fd07bd. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:30:45.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:44.234310 29224 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:45.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:45.636585 29222 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 29216
14:30:45.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:45.637380 29216 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.404s	user 0.521s	sys 0.866s
14:30:45.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:45.639038 29216 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.406s	user 0.521s	sys 0.866s
14:30:45.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:45.640543 29226 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:45.643181 29225 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1402 milliseconds
14:30:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.643260 29216 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:45.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.644490 29216 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.646733 29216 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:45.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.648083 29216 hybrid_clock.cc:648] HybridClock initialized: now 1783521045648073 us; error 68 us; skew 500 ppm
14:30:45.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.648942 29216 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:45.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.655148 29216 webserver.cc:533] Webserver started at http://127.26.209.194:35263/ using document root <none> and password file <none>
14:30:45.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.656237 29216 fs_manager.cc:362] Metadata directory not provided
14:30:45.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.656476 29216 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:45.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.664234 29216 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
14:30:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.668736 29244 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:45.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.669871 29216 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.670171 29216 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:14 on dist-test-slave-rt66"
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.672107 29216 fs_report.cc:389] FS layout report
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/data
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:45.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.721976 29216 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:45.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.723495 29216 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:45.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.723915 29216 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.726630 29216 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:45.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.732388 29251 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:45.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.740942 29216 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:45.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.741174 29216 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
14:30:45.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.741397 29216 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:45.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.746414 29216 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:45.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.746690 29216 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
14:30:45.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.747141 29251 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap starting.
14:30:45.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.816460 29251 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Log is configured to *not* fsync() on all Append() calls
14:30:45.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.898927 29251 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:45.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.899785 29251 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap complete.
14:30:45.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.901053 29251 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent bootstrapping tablet: real 0.154s	user 0.108s	sys 0.043s
14:30:45.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.918215 29251 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:45.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.921017 29251 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Initialized, Role: FOLLOWER
14:30:45.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.922223 29251 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:45.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.923420 29216 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.194:44581
14:30:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.923662 29358 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.194:44581 every 8 connection(s)
14:30:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.925920 29251 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent starting tablet: real 0.025s	user 0.023s	sys 0.001s
14:30:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.927008 29216 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:45.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.930277 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 29216
14:30:45.931 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:45.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.979306 29361 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:45.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.979842 29361 heartbeater.cc:461] Registering TS with master...
14:30:45.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.981305 29361 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.984522 29360 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.986972 27583 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:45.988 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.988934 29360 heartbeater.cc:461] Registering TS with master...
14:30:45.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.990146 29360 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:45.990 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.193:36347
14:30:45.990 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:30:45.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.991976 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 28852
14:30:45.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.994836 27664 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:45.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:45.997439 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.194:57231
14:30:46.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:46.001539 29362 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:46.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:46.001874 29362 heartbeater.cc:461] Registering TS with master...
14:30:46.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:46.002378 29362 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:46.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:46.005518 27508 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:46.020 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] handling channelInactive
14:30:46.020 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state READY due to: connection closed
14:30:46.021 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:46.022 [DEBUG - Time-limited test] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:46.021662 29373 negotiation.cc:336] Failed RPC negotiation. Trace:
14:30:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:45.990349 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.26.209.193:44803 (local address 127.26.209.194:44581)
14:30:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:45.993952 (+  3603us) server_negotiation.cc:207] Beginning negotiation
14:30:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:45.993984 (+    32us) server_negotiation.cc:400] Waiting for connection header
14:30:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:46.020025 (+ 26041us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.26.209.193:44803: BlockingRecv error: recv got EOF from 127.26.209.193:44803 (error 108)
14:30:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2976,"thread_start_us":3160,"threads_started":1}
14:30:46.023 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.023 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.023 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:46.041 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:46.041 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [19ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@629856930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@63b622e2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4586c0ff -> wakeup thread Time-limited test)) after lookup
14:30:46.041 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:46.042 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.042 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.043 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.060 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:46.060 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:46.060 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [19ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [20ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [22ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [22ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [38ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@629856930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@63b622e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4586c0ff -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883))) after lookup
14:30:46.061 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:46.062 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.062 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.063 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.100 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:46.100 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:46.101 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [19ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [20ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [22ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [22ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [38ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [39ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [41ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [42ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [79ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@629856930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@63b622e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4586c0ff -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207))) after lookup
14:30:46.101 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:46.102 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.102 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.103 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.200 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:46.200 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:46.201 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=179), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [19ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [20ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [22ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [22ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [38ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [39ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [41ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [42ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [79ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [80ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [82ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [179ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@629856930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@63b622e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207) -> (continuation of Deferred@1895544225 after retry RPC@1498414546), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4586c0ff -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207) -> (continuation of Deferred@1895544225 after retry RPC@1498414546))) after lookup
14:30:46.201 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:46.202 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.202 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.202 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.460 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:46.460 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:46.460 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=439), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [19ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [20ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [22ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [22ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [38ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [39ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [41ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [42ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [79ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [80ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [82ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [179ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [180ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [181ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [181ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [434ms] refreshing cache from master, [435ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [438ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@629856930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@63b622e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207) -> (continuation of Deferred@1895544225 after retry RPC@1498414546) -> (continuation of Deferred@256166605 after retry RPC@700647322), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4586c0ff -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207) -> (continuation of Deferred@1895544225 after retry RPC@1498414546) -> (continuation of Deferred@256166605 after retry RPC@700647322))) after lookup
14:30:46.461 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:46.462 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.462 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:46.462 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.001765 29360 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:47.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.285110 29376 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:30:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.285703 29376 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.288978 29376 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:47.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:47.294126 29247 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:47.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:47.299124 29247 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:47.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.303611 29116 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03" is_pre_election: true
14:30:47.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.305392 29245 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd; no voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03
14:30:47.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.306277 29376 raft_consensus.cc:2749] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
14:30:47.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.472579 29381 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 71108c6bdadf491b98e0d0b98c56682a)
14:30:47.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.473243 29381 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:47.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.476677 29381 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:47.492854 29049 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:47.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.497943 29314 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" is_pre_election: true
14:30:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.498965 29314 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 6.
14:30:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:47.499265 29049 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.501101 29049 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a
14:30:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.502102 29381 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 7 FOLLOWER]: Leader pre-election won for term 8
14:30:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.502486 29381 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 71108c6bdadf491b98e0d0b98c56682a)
14:30:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.502858 29381 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 7 FOLLOWER]: Advancing to term 8
14:30:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.510635 29381 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:47.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.513037 29381 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 8 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.514971 29314 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.515672 29314 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 6 FOLLOWER]: Advancing to term 8
14:30:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:47.518056 29049 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:30:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.523746 29314 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 8.
14:30:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.524978 29049 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a
14:30:47.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.525800 29381 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Leader election won for term 8
14:30:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.527855 29381 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 LEADER]: Becoming Leader. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Running, Role: LEADER
14:30:47.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.529270 29381 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:47.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:47.538657 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 reported cstate change: term changed from 7 to 8, leader changed from 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193) to cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195). New cstate: current_term: 8 leader_uuid: "cdc191ea1f6b4308a5a56e280893ec03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: HEALTHY } } }
14:30:48.021 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:48.021 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:48.021 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2000), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [19ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [20ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [22ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [22ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [38ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [39ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [41ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [42ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [79ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [80ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [82ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [179ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [180ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [181ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [181ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [434ms] refreshing cache from master, [435ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [438ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [439ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [441ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [441ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [1995ms] refreshing cache from master, [1995ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [2000ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@629856930(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@63b622e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207) -> (continuation of Deferred@1895544225 after retry RPC@1498414546) -> (continuation of Deferred@256166605 after retry RPC@700647322) -> (continuation of Deferred@1051641657 after retry RPC@258013345), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4586c0ff -> wakeup thread Time-limited test -> (continuation of Deferred@672777799 after retry RPC@1542154883) -> (continuation of Deferred@717208337 after retry RPC@713057207) -> (continuation of Deferred@1895544225 after retry RPC@1498414546) -> (continuation of Deferred@256166605 after retry RPC@700647322) -> (continuation of Deferred@1051641657 after retry RPC@258013345))) after lookup
14:30:48.022 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:48.023 [DEBUG - kudu-nio-7] (Connection.java:757) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] Successfully connected to peer
14:30:48.055 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0xa6b5589a, L:/127.0.0.1:44772 - R:/127.26.209.195:44447] using TOKEN/null
14:30:48.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:48.069170 29314 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 8 FOLLOWER]: Refusing update from remote peer cdc191ea1f6b4308a5a56e280893ec03: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
14:30:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:48.075224 29403 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.002s
14:30:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.076076 29049 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 -> Peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Couldn't send request to peer 71108c6bdadf491b98e0d0b98c56682a. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:48.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:48.092476 29407 mvcc.cc:204] Tried to move back new op lower bound from 7305302195107586048 to 7305302188500512768. Current Snapshot: MvccSnapshot[applied={T|T < 7305302195107586048}]
14:30:48.147 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.193:36347
14:30:48.148 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:48.149583 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:36347
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45851
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:48.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.464074 29414 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:48.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.464557 29414 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.465068 29414 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.499626 29414 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:48.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.500602 29414 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.193
14:30:48.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:48.540419 29414 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:36347
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45851
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:48.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:48.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:48.542163 29414 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:48.543887 29414 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.558151 29420 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:49.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:48.561100 29421 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:49.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:49.749008 29423 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:49.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:49.751084 29422 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1186 milliseconds
14:30:49.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.751302 29414 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:49.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.753086 29414 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.756074 29414 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:49.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.757565 29414 hybrid_clock.cc:648] HybridClock initialized: now 1783521049757484 us; error 72 us; skew 500 ppm
14:30:49.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.758384 29414 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.765918 29414 webserver.cc:533] Webserver started at http://127.26.209.193:45851/ using document root <none> and password file <none>
14:30:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.767318 29414 fs_manager.cc:362] Metadata directory not provided
14:30:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.767633 29414 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:49.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.776316 29414 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.000s
14:30:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.781252 29435 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.782384 29414 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.004s
14:30:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.782701 29414 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:13 on dist-test-slave-rt66"
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.784596 29414 fs_report.cc:389] FS layout report
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/data
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.856182 29414 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.858335 29414 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:49.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.858922 29414 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.862975 29414 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.871104 29442 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:49.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.880561 29414 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:49.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.880935 29414 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
14:30:49.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.881299 29414 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:49.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.889667 29414 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:49.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.889961 29414 ts_tablet_manager.cc:595] Time spent register tablets: real 0.009s	user 0.006s	sys 0.002s
14:30:49.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.890435 29442 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap starting.
14:30:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:49.967149 29442 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Log is configured to *not* fsync() on all Append() calls
14:30:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.077049 29442 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.078804 29442 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap complete.
14:30:50.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.080682 29442 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent bootstrapping tablet: real 0.191s	user 0.124s	sys 0.051s
14:30:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.102254 29442 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.106072 29442 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 71108c6bdadf491b98e0d0b98c56682a, State: Initialized, Role: FOLLOWER
14:30:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.107990 29442 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.113998 29442 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent starting tablet: real 0.033s	user 0.031s	sys 0.000s
14:30:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.140479 29414 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.193:36347
14:30:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.140815 29550 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.193:36347 every 8 connection(s)
14:30:50.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.143231 29414 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:30:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.153697 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 29414
14:30:50.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:50.205 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:50.206 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.195:44447
14:30:50.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:30:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.207094 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 29027
14:30:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.229924 29553 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.230050 29552 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.230468 29553 heartbeater.cc:461] Registering TS with master...
14:30:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.230858 29552 heartbeater.cc:461] Registering TS with master...
14:30:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.231874 29553 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.232091 29552 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.234350 29551 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.234726 29551 heartbeater.cc:461] Registering TS with master...
14:30:50.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.235603 29551 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:50.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.238876 27664 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:50.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.239789 27583 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.242971 27509 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:30:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:50.243031 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.193:58363
14:30:50.248 [DEBUG - kudu-nio-7] (Connection.java:239) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] handling channelInactive
14:30:50.249 [DEBUG - kudu-nio-7] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state READY due to: connection closed
14:30:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:50.251160 29563 negotiation.cc:336] Failed RPC negotiation. Trace:
14:30:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.216587 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.26.209.195:56627 (local address 127.26.209.193:36347)
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.226702 (+ 10115us) server_negotiation.cc:207] Beginning negotiation
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.226732 (+    30us) server_negotiation.cc:400] Waiting for connection header
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.226785 (+    53us) server_negotiation.cc:408] Connection header received
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.226974 (+   189us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.227010 (+    36us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.227373 (+   363us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.227577 (+   204us) server_negotiation.cc:227] Negotiated authn=SASL
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:50.249847 (+ 22270us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.26.209.195:56627: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":9474,"thread_start_us":9115,"threads_started":1}
14:30:50.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:50.254 [DEBUG - Time-limited test] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:50.255 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.255 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.256 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.281 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:50.281 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:50.281 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [3ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@58354208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@4fe126d7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@37440b7e -> wakeup thread Time-limited test)) after lookup
14:30:50.282 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:50.283 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.283 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.283 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.299 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:50.299 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:50.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [3ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [30ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [46ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@58354208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@4fe126d7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@37440b7e -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161))) after lookup
14:30:50.300 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:50.301 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.301 [DEBUG - kudu-nio-2] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.301 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.340 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:50.341 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:50.341 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [3ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [30ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [46ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [47ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [48ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [48ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@58354208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@4fe126d7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@37440b7e -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795))) after lookup
14:30:50.341 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:50.342 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.343 [DEBUG - kudu-nio-3] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.343 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.420 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:50.421 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:50.421 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=167), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [3ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [30ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [46ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [47ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [48ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [48ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [90ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [90ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [167ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@58354208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@4fe126d7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@37440b7e -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378))) after lookup
14:30:50.421 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:50.422 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.423 [DEBUG - kudu-nio-4] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.423 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.639 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:50.640 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:50.640 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=386), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [3ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [30ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [46ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [47ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [48ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [48ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [90ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [90ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [167ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [168ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [170ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [170ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [382ms] refreshing cache from master, [382ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [386ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@58354208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@4fe126d7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378) -> (continuation of Deferred@641236020 after retry RPC@2050836801), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@37440b7e -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378) -> (continuation of Deferred@641236020 after retry RPC@2050836801))) after lookup
14:30:50.641 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:50.642 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.642 [DEBUG - kudu-nio-5] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.642 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.740 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:50.740 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:30:50.740 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=487), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [3ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [30ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [46ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [47ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [48ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [48ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [90ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [90ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [167ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [168ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [170ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [170ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [382ms] refreshing cache from master, [382ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [386ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [387ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [389ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [389ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [483ms] refreshing cache from master, [483ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [486ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@58354208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@4fe126d7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378) -> (continuation of Deferred@641236020 after retry RPC@2050836801) -> (continuation of Deferred@1334483613 after retry RPC@1601021292), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@37440b7e -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378) -> (continuation of Deferred@641236020 after retry RPC@2050836801) -> (continuation of Deferred@1334483613 after retry RPC@1601021292))) after lookup
14:30:50.742 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:30:50.743 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.743 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:50.743 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447
14:30:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.247166 29551 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.479040 29564 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 8 FOLLOWER]: Starting pre-election (detected failure of leader cdc191ea1f6b4308a5a56e280893ec03)
14:30:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.479455 29564 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:51.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.481161 29564 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:51.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:51.502377 29245 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:51.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.508528 29505 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a" is_pre_election: true
14:30:51.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.509413 29505 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 7.
14:30:51.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.510838 29247 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd; no voters: cdc191ea1f6b4308a5a56e280893ec03
14:30:51.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.511613 29564 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 8 FOLLOWER]: Leader pre-election won for term 9
14:30:51.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.511957 29564 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 8 FOLLOWER]: Starting leader election (detected failure of leader cdc191ea1f6b4308a5a56e280893ec03)
14:30:51.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.512287 29564 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 8 FOLLOWER]: Advancing to term 9
14:30:51.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.520468 29564 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:51.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.523279 29564 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 9 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:51.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.524204 29505 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:30:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.524740 29505 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 7 FOLLOWER]: Advancing to term 9
14:30:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:51.527312 29245 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111)
14:30:51.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.532375 29505 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4522c04ae7349348eaf3cf1d3fd07bd in term 9.
14:30:51.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.533452 29247 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd; no voters: cdc191ea1f6b4308a5a56e280893ec03
14:30:51.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.534103 29564 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 9 FOLLOWER]: Leader election won for term 9
14:30:51.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.535202 29564 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 9 LEADER]: Becoming Leader. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Running, Role: LEADER
14:30:51.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.536077 29564 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.544755 27664 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd reported cstate change: term changed from 8 to 9, leader changed from cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195) to b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194). New cstate: current_term: 9 leader_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: UNKNOWN } } }
14:30:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.943049 29505 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 9 FOLLOWER]: Refusing update from remote peer b4522c04ae7349348eaf3cf1d3fd07bd: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
14:30:51.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:51.944671 29564 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [LEADER]: Connected to new peer: Peer: permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
14:30:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:51.981803 29245 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd -> Peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Couldn't send request to peer cdc191ea1f6b4308a5a56e280893ec03. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:52.379 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:52.380 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:52.380 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2126), Traces: [0ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [2ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [3ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [30ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [30ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [46ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [47ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [48ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [48ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [87ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [90ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [90ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [167ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [168ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [170ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [170ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [382ms] refreshing cache from master, [382ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [386ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [387ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [389ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [389ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [483ms] refreshing cache from master, [483ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [486ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [488ms] sending RPC to server cdc191ea1f6b4308a5a56e280893ec03, [490ms] received response from server cdc191ea1f6b4308a5a56e280893ec03: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [490ms] delaying RPC due to: Network error: Failed to connect to peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447): Connection refused: /127.26.209.195:44447, [2122ms] refreshing cache from master, [2123ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [2126ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@58354208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@4fe126d7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378) -> (continuation of Deferred@641236020 after retry RPC@2050836801) -> (continuation of Deferred@1334483613 after retry RPC@1601021292) -> (continuation of Deferred@2108129667 after retry RPC@1330847134), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@37440b7e -> wakeup thread Time-limited test -> (continuation of Deferred@993146570 after retry RPC@742053161) -> (continuation of Deferred@214056679 after retry RPC@1990926795) -> (continuation of Deferred@772895034 after retry RPC@26978378) -> (continuation of Deferred@641236020 after retry RPC@2050836801) -> (continuation of Deferred@1334483613 after retry RPC@1601021292) -> (continuation of Deferred@2108129667 after retry RPC@1330847134))) after lookup
14:30:52.380 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:52.382 [DEBUG - kudu-nio-7] (Connection.java:757) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] Successfully connected to peer
14:30:52.399 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x4950f60c, L:/127.0.0.1:42896 - R:/127.26.209.194:44581] using TOKEN/null
14:30:52.445 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.195:44447
14:30:52.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.447122 29245 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111) [suppressed 3 similar messages]
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:52.447230 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:44447
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39951
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:52.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.764665 29593 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:52.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.765206 29593 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:52.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.765688 29593 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:52.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.799297 29593 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:52.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.800220 29593 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.195
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:52.837551 29593 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.195:44447
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.195
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39951
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.195
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/logs
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:52.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:52.839010 29593 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:52.840636 29593 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.854249 29599 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.327028 29245 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd -> Peer cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447): Couldn't send request to peer cdc191ea1f6b4308a5a56e280893ec03. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.195:44447: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:30:54.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.257309 29598 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 29593
14:30:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.394850 29598 kernel_stack_watchdog.cc:198] Thread 29593 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 400ms:
14:30:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:30:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:30:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:30:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:30:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:52.854776 29600 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.395414 29593 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.541s	user 0.558s	sys 0.959s
14:30:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.395763 29593 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.542s	user 0.558s	sys 0.959s
14:30:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.397678 29602 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:54.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.400949 29601 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1541 milliseconds
14:30:54.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.401062 29593 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:54.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.402302 29593 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.404490 29593 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:54.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.405874 29593 hybrid_clock.cc:648] HybridClock initialized: now 1783521054405871 us; error 88 us; skew 500 ppm
14:30:54.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.406672 29593 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.412945 29593 webserver.cc:533] Webserver started at http://127.26.209.195:39951/ using document root <none> and password file <none>
14:30:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.414062 29593 fs_manager.cc:362] Metadata directory not provided
14:30:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.414347 29593 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:54.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.422747 29593 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.004s
14:30:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.427721 29614 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:54.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.429108 29593 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
14:30:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.429440 29593 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:16 on dist-test-slave-rt66"
14:30:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.431422 29593 fs_report.cc:389] FS layout report
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/wal
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/data
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.478662 29593 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.480429 29593 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.480878 29593 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.483816 29593 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:54.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.493336 29621 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.506052 29593 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.506490 29593 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.017s	user 0.002s	sys 0.001s
14:30:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.507118 29593 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.515336 29593 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.515631 29593 ts_tablet_manager.cc:595] Time spent register tablets: real 0.009s	user 0.007s	sys 0.000s
14:30:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.516120 29621 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap starting.
14:30:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.587266 29621 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Log is configured to *not* fsync() on all Append() calls
14:30:54.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.683380 29621 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:54.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.684201 29621 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Bootstrap complete.
14:30:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.685562 29621 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent bootstrapping tablet: real 0.170s	user 0.103s	sys 0.063s
14:30:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.704361 29593 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.195:44447
14:30:54.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.704572 29728 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.195:44447 every 8 connection(s)
14:30:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.708022 29593 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-2/data/info.pb
14:30:54.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.709393 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 29593
14:30:54.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.709776 29621 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.712973 29621 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Initialized, Role: FOLLOWER
14:30:54.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.715132 29621 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:54.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.749330 29621 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03: Time spent starting tablet: real 0.064s	user 0.023s	sys 0.017s
14:30:54.764 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:54.765 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.194:44581
14:30:54.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.769090 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 29216
14:30:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.812932 29731 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.813391 29731 heartbeater.cc:461] Registering TS with master...
14:30:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.814520 29730 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.814615 29732 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.814745 29731 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.814922 29730 heartbeater.cc:461] Registering TS with master...
14:30:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.814922 29732 heartbeater.cc:461] Registering TS with master...
14:30:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.816040 29732 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.816097 29730 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.819633 27583 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.819937 27665 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.820343 27509 ts_manager.cc:194] Re-registered known tserver with Master: cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.822773 27665 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.195:55257
14:30:54.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:54.826534 29730 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:54.828 [DEBUG - kudu-nio-7] (Connection.java:239) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] handling channelInactive
14:30:54.828 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state READY due to: connection closed
14:30:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:54.829761 29742 negotiation.cc:336] Failed RPC negotiation. Trace:
14:30:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.806738 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.26.209.194:59087 (local address 127.26.209.195:44447)
14:30:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.812525 (+  5787us) server_negotiation.cc:207] Beginning negotiation
14:30:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.812566 (+    41us) server_negotiation.cc:400] Waiting for connection header
14:30:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.812621 (+    55us) server_negotiation.cc:408] Connection header received
14:30:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.812862 (+   241us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:30:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.812901 (+    39us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:30:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.813275 (+   374us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:30:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.813473 (+   198us) server_negotiation.cc:227] Negotiated authn=SASL
14:30:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 14:30:54.827638 (+ 14165us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.26.209.194:59087: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:30:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4379,"thread_start_us":3350,"threads_started":1}
14:30:54.834 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:54.835 [DEBUG - Time-limited test] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:54.836 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.836 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.836 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.860 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:54.861 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:54.861 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [26ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1359357318(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@19bcd6b7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41720187 -> wakeup thread Time-limited test)) after lookup
14:30:54.861 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:54.862 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.863 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.863 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:54.880 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:54.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [26ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [27ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [29ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [29ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [45ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1359357318(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@19bcd6b7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41720187 -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871))) after lookup
14:30:54.880 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:54.882 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.882 [DEBUG - kudu-nio-2] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.882 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.921 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:54.921 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:54.921 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [26ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [27ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [29ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [29ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [45ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [46ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [48ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1359357318(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@19bcd6b7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41720187 -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490))) after lookup
14:30:54.922 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:54.923 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.923 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:54.924 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:55.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:55.040 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:55.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=206), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [26ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [27ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [29ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [29ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [45ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [46ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [48ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [89ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [201ms] refreshing cache from master, [202ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [205ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1359357318(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@19bcd6b7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490) -> (continuation of Deferred@243062361 after retry RPC@693352834), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41720187 -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490) -> (continuation of Deferred@243062361 after retry RPC@693352834))) after lookup
14:30:55.041 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:55.042 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:55.042 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:55.042 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:55.401 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:55.401 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)[L],cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:55.401 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=567), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [26ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [27ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [29ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [29ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [45ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [46ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [48ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [89ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [201ms] refreshing cache from master, [202ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [205ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [207ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [208ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [208ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [562ms] refreshing cache from master, [562ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [566ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1359357318(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@19bcd6b7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490) -> (continuation of Deferred@243062361 after retry RPC@693352834) -> (continuation of Deferred@192053118 after retry RPC@1062819899), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41720187 -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490) -> (continuation of Deferred@243062361 after retry RPC@693352834) -> (continuation of Deferred@192053118 after retry RPC@1062819899))) after lookup
14:30:55.402 [DEBUG - kudu-nio-0] (Connection.java:748) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] connecting to peer
14:30:55.403 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:55.403 [DEBUG - kudu-nio-5] (Connection.java:709) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] cleaning up while in state CONNECTING due to: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:55.403 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581
14:30:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.005771 29744 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:30:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.006295 29744 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:56.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.009119 29744 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:56.014605 29617 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:56.019697 29617 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.021597 29505 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "71108c6bdadf491b98e0d0b98c56682a" is_pre_election: true
14:30:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.023490 29617 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a, b4522c04ae7349348eaf3cf1d3fd07bd
14:30:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.024596 29744 raft_consensus.cc:2749] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
14:30:56.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.203994 29749 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 9 FOLLOWER]: Starting pre-election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.204752 29749 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.207566 29749 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:56.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:56.210335 29438 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:56.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:56.215309 29438 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.220559 29684 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03" is_pre_election: true
14:30:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.221325 29684 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 71108c6bdadf491b98e0d0b98c56682a in term 8.
14:30:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.223824 29436 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:56.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.225178 29749 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 9 FOLLOWER]: Leader pre-election won for term 10
14:30:56.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.225755 29749 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 9 FOLLOWER]: Starting leader election (detected failure of leader b4522c04ae7349348eaf3cf1d3fd07bd)
14:30:56.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.226109 29749 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 9 FOLLOWER]: Advancing to term 10
14:30:56.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.234212 29749 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:56.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.237355 29749 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 10 election: Requested vote from peers b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581), cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195:44447)
14:30:56.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.238859 29684 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "71108c6bdadf491b98e0d0b98c56682a" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "cdc191ea1f6b4308a5a56e280893ec03"
14:30:56.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.239418 29684 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 8 FOLLOWER]: Advancing to term 10
14:30:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:56.241555 29438 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111)
14:30:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.246078 29684 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 71108c6bdadf491b98e0d0b98c56682a in term 10.
14:30:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.247094 29436 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 71108c6bdadf491b98e0d0b98c56682a, cdc191ea1f6b4308a5a56e280893ec03; no voters: b4522c04ae7349348eaf3cf1d3fd07bd
14:30:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.247745 29749 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 10 FOLLOWER]: Leader election won for term 10
14:30:56.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.249444 29749 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 10 LEADER]: Becoming Leader. State: Replica: 71108c6bdadf491b98e0d0b98c56682a, State: Running, Role: LEADER
14:30:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.250437 29749 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.259724 27665 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a reported cstate change: term changed from 9 to 10, leader changed from b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194) to 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193). New cstate: current_term: 10 leader_uuid: "71108c6bdadf491b98e0d0b98c56682a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: UNKNOWN } } }
14:30:56.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.680527 29684 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 10 FOLLOWER]: Refusing update from remote peer 71108c6bdadf491b98e0d0b98c56682a: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
14:30:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:56.682294 29749 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
14:30:56.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:56.727147 29438 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a -> Peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Couldn't send request to peer b4522c04ae7349348eaf3cf1d3fd07bd. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:30:57.080 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:57.080 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:57.081 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2246), Traces: [0ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [2ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [26ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [27ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [29ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [29ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [45ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [46ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [48ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [48ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [89ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [90ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [201ms] refreshing cache from master, [202ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [205ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [207ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [208ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [208ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [562ms] refreshing cache from master, [562ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [566ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [568ms] sending RPC to server b4522c04ae7349348eaf3cf1d3fd07bd, [569ms] received response from server b4522c04ae7349348eaf3cf1d3fd07bd: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [569ms] delaying RPC due to: Network error: Failed to connect to peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581): Connection refused: /127.26.209.194:44581, [2242ms] refreshing cache from master, [2242ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [2246ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1359357318(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@19bcd6b7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490) -> (continuation of Deferred@243062361 after retry RPC@693352834) -> (continuation of Deferred@192053118 after retry RPC@1062819899) -> (continuation of Deferred@1542965332 after retry RPC@781900689), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@41720187 -> wakeup thread Time-limited test -> (continuation of Deferred@330378471 after retry RPC@1117677871) -> (continuation of Deferred@1775844839 after retry RPC@1488971490) -> (continuation of Deferred@243062361 after retry RPC@693352834) -> (continuation of Deferred@192053118 after retry RPC@1062819899) -> (continuation of Deferred@1542965332 after retry RPC@781900689))) after lookup
14:30:57.081 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:57.082 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] Successfully connected to peer
14:30:57.113 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0x60cee4ae, L:/127.0.0.1:52114 - R:/127.26.209.193:36347] using TOKEN/null
14:30:57.162 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.194:44581
14:30:57.163 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:57.164259 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:30:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:57.489820 29777 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:30:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:57.490475 29777 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:30:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:57.491252 29777 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:30:57.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:57.536305 29777 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:30:57.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:57.537292 29777 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.194
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:57.576661 29777 tablet_server_runner.cc:78] Tablet server non-default flags:
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.194:44581
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.194
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35263
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:30:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.194
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/logs
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:30:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:57.578531 29777 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:30:57.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:57.580602 29777 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:30:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:57.594965 29786 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:58.997730 29785 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 29777
14:30:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:59.044062 29785 kernel_stack_watchdog.cc:198] Thread 29777 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
14:30:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:30:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:30:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:30:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:30:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:59.044584 29777 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.449s	user 0.571s	sys 0.806s
14:30:59.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:59.045003 29777 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.450s	user 0.571s	sys 0.806s
14:30:59.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:57.596125 29787 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:59.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:59.047725 29789 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:30:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:59.052001 29788 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1448 milliseconds
14:30:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.052156 29777 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:30:59.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.053517 29777 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:30:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.055967 29777 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:30:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.057374 29777 hybrid_clock.cc:648] HybridClock initialized: now 1783521059057345 us; error 46 us; skew 500 ppm
14:30:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.058199 29777 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:30:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.064498 29777 webserver.cc:533] Webserver started at http://127.26.209.194:35263/ using document root <none> and password file <none>
14:30:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.065532 29777 fs_manager.cc:362] Metadata directory not provided
14:30:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.065749 29777 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:30:59.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.074154 29777 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.000s
14:30:59.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.078962 29800 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:30:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.080351 29777 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
14:30:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.080752 29777 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:30:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:14 on dist-test-slave-rt66"
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.082823 29777 fs_report.cc:389] FS layout report
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/wal
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/data
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:30:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:30:59.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.144459 29777 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:30:59.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.146190 29777 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:30:59.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.146682 29777 kserver.cc:163] Server-wide thread pool size limit: 3276
14:30:59.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.150291 29777 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:30:59.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.157612 29807 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:30:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.166882 29777 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:30:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.167204 29777 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
14:30:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.167526 29777 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:30:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.172878 29777 ts_tablet_manager.cc:616] Registered 1 tablets
14:30:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.173130 29777 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.006s	sys 0.000s
14:30:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.173720 29807 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap starting.
14:30:59.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.249570 29807 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Log is configured to *not* fsync() on all Append() calls
14:30:59.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:30:59.314244 29438 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a -> Peer b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581): Couldn't send request to peer b4522c04ae7349348eaf3cf1d3fd07bd. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.194:44581: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:30:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.356353 29777 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.194:44581
14:30:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.356516 29916 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.194:44581 every 8 connection(s)
14:30:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.358974 29777 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-1/data/info.pb
14:30:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.361434 29807 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:30:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.362416 29807 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Bootstrap complete.
14:30:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.364107 29807 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent bootstrapping tablet: real 0.191s	user 0.139s	sys 0.045s
14:30:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.364646 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 29777
14:30:59.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:59.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.388209 29807 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:59.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.391855 29807 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4522c04ae7349348eaf3cf1d3fd07bd, State: Initialized, Role: FOLLOWER
14:30:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.392951 29807 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:30:59.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.403088 29807 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd: Time spent starting tablet: real 0.039s	user 0.034s	sys 0.005s
14:30:59.405 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:30:59.406 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.26.209.193:36347
14:30:59.406 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:30:59.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.406898 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 29414
14:30:59.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.432673 29917 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:30:59.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.433226 29917 heartbeater.cc:461] Registering TS with master...
14:30:59.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.434553 29917 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:30:59.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.435283 29918 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:30:59.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.435654 29918 heartbeater.cc:461] Registering TS with master...
14:30:59.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.436632 29918 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:30:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.438771 27665 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.440503 27583 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.442060 27665 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.194:40041
14:30:59.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.445576 29919 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:30:59.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.445955 29919 heartbeater.cc:461] Registering TS with master...
14:30:59.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.447376 29919 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:30:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.449237 29917 heartbeater.cc:499] Master 127.26.209.252:41417 was elected leader, sending a full tablet report...
14:30:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:30:59.450799 27509 ts_manager.cc:194] Re-registered known tserver with Master: b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:30:59.452 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] handling channelInactive
14:30:59.452 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state READY due to: connection closed
14:30:59.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:30:59.453 [DEBUG - Time-limited test] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:59.455 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.455 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.455 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.481 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:59.481 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:59.481 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1256845906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@32d7eecd -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f8a1f8d -> wakeup thread Time-limited test)) after lookup
14:30:59.481 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:59.482 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.482 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.483 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.500 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:59.501 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:59.501 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [29ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [30ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [47ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1256845906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@32d7eecd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f8a1f8d -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994))) after lookup
14:30:59.502 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:59.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.511 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.540 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:59.540 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:59.541 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [29ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [30ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [47ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [48ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [58ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [58ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1256845906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@32d7eecd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f8a1f8d -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270))) after lookup
14:30:59.542 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:59.543 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.543 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.543 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.600 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:59.600 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:59.600 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=147), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [29ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [30ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [47ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [48ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [58ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [58ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [90ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [90ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [147ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1256845906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@32d7eecd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f8a1f8d -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035))) after lookup
14:30:59.601 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:59.602 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.602 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.602 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:30:59.842 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:30:59.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=389), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [29ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [30ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [47ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [48ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [58ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [58ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [90ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [90ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [147ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [148ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [149ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [149ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [383ms] refreshing cache from master, [383ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [388ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1256845906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@32d7eecd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035) -> (continuation of Deferred@682717395 after retry RPC@2093963265), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f8a1f8d -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035) -> (continuation of Deferred@682717395 after retry RPC@2093963265))) after lookup
14:30:59.843 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:30:59.844 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.845 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:30:59.845 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:31:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.397886 29929 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 71108c6bdadf491b98e0d0b98c56682a)
14:31:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.398324 29929 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:31:00.400 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:31:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.400404 29929 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:31:00.401 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)[L],b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)]}]
14:31:00.401 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=947), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [29ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [30ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [47ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [48ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [58ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [58ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [90ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [90ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [147ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [148ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [149ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [149ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [383ms] refreshing cache from master, [383ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [388ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [390ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [392ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [392ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [943ms] refreshing cache from master, [943ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [947ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1256845906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@32d7eecd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035) -> (continuation of Deferred@682717395 after retry RPC@2093963265) -> (continuation of Deferred@1279773035 after retry RPC@1255055282), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f8a1f8d -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035) -> (continuation of Deferred@682717395 after retry RPC@2093963265) -> (continuation of Deferred@1279773035 after retry RPC@1255055282))) after lookup
14:31:00.402 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] connecting to peer
14:31:00.404 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:31:00.404 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] cleaning up while in state CONNECTING due to: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:31:00.404 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347) for tablet 13b7e17c89d24bf7933f32c1ea7c621f: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347
14:31:00.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:00.411993 29617 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:31:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.416457 29870 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" is_pre_election: true
14:31:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.417342 29870 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 9.
14:31:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.418730 29617 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a
14:31:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.419929 29929 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 10 FOLLOWER]: Leader pre-election won for term 11
14:31:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.420250 29929 raft_consensus.cc:493] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 71108c6bdadf491b98e0d0b98c56682a)
14:31:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.420553 29929 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 10 FOLLOWER]: Advancing to term 11
14:31:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.427831 29929 raft_consensus.cc:515] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:31:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.429824 29929 leader_election.cc:290] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 11 election: Requested vote from peers 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347), b4522c04ae7349348eaf3cf1d3fd07bd (127.26.209.194:44581)
14:31:00.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.433135 29870 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "13b7e17c89d24bf7933f32c1ea7c621f" candidate_uuid: "cdc191ea1f6b4308a5a56e280893ec03" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd"
14:31:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.434713 29870 raft_consensus.cc:3060] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 9 FOLLOWER]: Advancing to term 11
14:31:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:00.435274 29617 leader_election.cc:336] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111)
14:31:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.446241 29870 raft_consensus.cc:2468] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdc191ea1f6b4308a5a56e280893ec03 in term 11.
14:31:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.447921 29617 leader_election.cc:304] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b4522c04ae7349348eaf3cf1d3fd07bd, cdc191ea1f6b4308a5a56e280893ec03; no voters: 71108c6bdadf491b98e0d0b98c56682a
14:31:00.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.449060 29929 raft_consensus.cc:2804] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 11 FOLLOWER]: Leader election won for term 11
14:31:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.451593 29929 raft_consensus.cc:697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [term 11 LEADER]: Becoming Leader. State: Replica: cdc191ea1f6b4308a5a56e280893ec03, State: Running, Role: LEADER
14:31:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.452976 29929 consensus_queue.cc:237] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:31:00.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.475924 27665 catalog_manager.cc:5697] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 reported cstate change: term changed from 10 to 11, leader changed from 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193) to cdc191ea1f6b4308a5a56e280893ec03 (127.26.209.195). New cstate: current_term: 11 leader_uuid: "cdc191ea1f6b4308a5a56e280893ec03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } health_report { overall_health: HEALTHY } } }
14:31:00.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:00.929579 29617 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 -> Peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Couldn't send request to peer 71108c6bdadf491b98e0d0b98c56682a. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:31:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.984807 29870 raft_consensus.cc:1275] T 13b7e17c89d24bf7933f32c1ea7c621f P b4522c04ae7349348eaf3cf1d3fd07bd [term 11 FOLLOWER]: Refusing update from remote peer cdc191ea1f6b4308a5a56e280893ec03: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
14:31:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:00.986371 29929 consensus_queue.cc:1048] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
14:31:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:01.507656 29617 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111) [suppressed 3 similar messages]
14:31:02.981 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 13b7e17c89d24bf7933f32c1ea7c621f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622' with partition [<start>, <end>)
14:31:02.981 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=13b7e17c89d24bf7933f32c1ea7c621f@[71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347),b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581),cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)[L]]}]
14:31:02.982 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=13b7e17c89d24bf7933f32c1ea7c621f, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=3528), Traces: [0ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [2ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [2ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [27ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [28ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [29ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [30ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [47ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [48ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [58ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [58ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [82ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [86ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [88ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [90ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [90ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [142ms] refreshing cache from master, [143ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [147ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [148ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [149ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [149ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [383ms] refreshing cache from master, [383ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [388ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [390ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [392ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [392ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [943ms] refreshing cache from master, [943ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [947ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, [949ms] sending RPC to server 71108c6bdadf491b98e0d0b98c56682a, [951ms] received response from server 71108c6bdadf491b98e0d0b98c56682a: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [951ms] delaying RPC due to: Network error: Failed to connect to peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347): Connection refused: /127.26.209.193:36347, [3523ms] refreshing cache from master, [3523ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.209.252:41417, [3528ms] Sub RPC GetTableLocations: received response from server master-127.26.209.252:41417: OK, deferred=Deferred@1256845906(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007f7d5036b7a8@32d7eecd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035) -> (continuation of Deferred@682717395 after retry RPC@2093963265) -> (continuation of Deferred@1279773035 after retry RPC@1255055282) -> (continuation of Deferred@1205482415 after retry RPC@2035746793), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f8a1f8d -> wakeup thread Time-limited test -> (continuation of Deferred@1528169970 after retry RPC@1272013994) -> (continuation of Deferred@928192026 after retry RPC@2003292270) -> (continuation of Deferred@964283442 after retry RPC@1884590035) -> (continuation of Deferred@682717395 after retry RPC@2093963265) -> (continuation of Deferred@1279773035 after retry RPC@1255055282) -> (continuation of Deferred@1205482415 after retry RPC@2035746793))) after lookup
14:31:02.983 [DEBUG - kudu-nio-0] (Connection.java:748) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] connecting to peer
14:31:02.984 [DEBUG - kudu-nio-6] (Connection.java:757) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] Successfully connected to peer
14:31:03.007 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0xad3c9753, L:/127.0.0.1:58214 - R:/127.26.209.195:44447] using TOKEN/null
14:31:03.074 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.26.209.193:36347
14:31:03.074 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:03.075598 27463 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:36347
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45851
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:31:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:31:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.441282 29969 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:31:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.441992 29969 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:31:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.442682 29969 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:31:03.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.484391 29969 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:31:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.486012 29969 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.209.193
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:03.541471 29969 tablet_server_runner.cc:78] Tablet server non-default flags:
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.209.212:37509
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.209.193:36347
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.209.193
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45851
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.209.254:45405,127.26.209.253:34165,127.26.209.252:41417
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.209.193
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/logs
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:31:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e5736f718e5e433722f6b8766b4c965bea8722fd
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 14:12:00 UTC on bdcb31816ec0
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16864
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:31:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:03.543022 29969 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:31:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:03.544932 29969 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:31:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.559860 29975 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:31:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.664873 29617 consensus_peers.cc:597] T 13b7e17c89d24bf7933f32c1ea7c621f P cdc191ea1f6b4308a5a56e280893ec03 -> Peer 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347): Couldn't send request to peer 71108c6bdadf491b98e0d0b98c56682a. Status: Network error: Client connection negotiation failed: client connection to 127.26.209.193:36347: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:31:04.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:03.560204 29976 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:31:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:04.932462 29978 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:31:04.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:04.936285 29969 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.377s	user 0.511s	sys 0.840s
14:31:04.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:04.936697 29969 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.377s	user 0.513s	sys 0.842s
14:31:04.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 14:31:04.936940 29977 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1376 milliseconds
14:31:04.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.936997 29969 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:31:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.938601 29969 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:31:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.941524 29969 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:31:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.942987 29969 hybrid_clock.cc:648] HybridClock initialized: now 1783521064942836 us; error 131 us; skew 500 ppm
14:31:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.944108 29969 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:31:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.953058 29969 webserver.cc:533] Webserver started at http://127.26.209.193:45851/ using document root <none> and password file <none>
14:31:04.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.954370 29969 fs_manager.cc:362] Metadata directory not provided
14:31:04.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.954661 29969 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:31:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.965597 29969 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.002s
14:31:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.972342 29989 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
14:31:04.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.974040 29969 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.003s	sys 0.003s
14:31:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.974426 29969 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data,/tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:31:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71108c6bdadf491b98e0d0b98c56682a"
14:31:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 14:30:13 on dist-test-slave-rt66"
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:04.976665 29969 fs_report.cc:389] FS layout report
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/wal
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/data
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:31:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:31:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.060096 29969 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
14:31:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.062443 29969 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:31:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.063038 29969 kserver.cc:163] Server-wide thread pool size limit: 3276
14:31:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.066812 29969 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:31:05.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.073933 29996 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
14:31:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.082759 29969 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
14:31:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.083024 29969 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
14:31:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.083243 29969 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
14:31:05.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.088615 29969 ts_tablet_manager.cc:616] Registered 1 tablets
14:31:05.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.088881 29969 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.002s	sys 0.004s
14:31:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.089759 29996 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap starting.
14:31:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.168563 29996 log.cc:826] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Log is configured to *not* fsync() on all Append() calls
14:31:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.267019 29969 rpc_server.cc:307] RPC server started. Bound to: 127.26.209.193:36347
14:31:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.267210 30103 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.209.193:36347 every 8 connection(s)
14:31:05.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.270113 29969 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskksXv_G/test-tmp/mini-kudu-cluster16098729845554357046/ts-0/data/info.pb
14:31:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.277612 27463 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu as pid 29969
14:31:05.280 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
14:31:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.302863 29996 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:31:05.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.304003 29996 tablet_bootstrap.cc:492] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Bootstrap complete.
14:31:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.305967 29996 ts_tablet_manager.cc:1403] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent bootstrapping tablet: real 0.217s	user 0.198s	sys 0.017s
14:31:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.325719 29996 raft_consensus.cc:359] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:31:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.329448 29996 raft_consensus.cc:740] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 71108c6bdadf491b98e0d0b98c56682a, State: Initialized, Role: FOLLOWER
14:31:05.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.330749 29996 consensus_queue.cc:260] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "71108c6bdadf491b98e0d0b98c56682a" member_type: VOTER last_known_addr { host: "127.26.209.193" port: 36347 } } peers { permanent_uuid: "b4522c04ae7349348eaf3cf1d3fd07bd" member_type: VOTER last_known_addr { host: "127.26.209.194" port: 44581 } } peers { permanent_uuid: "cdc191ea1f6b4308a5a56e280893ec03" member_type: VOTER last_known_addr { host: "127.26.209.195" port: 44447 } }
14:31:05.337 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:31:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.340912 29996 ts_tablet_manager.cc:1434] T 13b7e17c89d24bf7933f32c1ea7c621f P 71108c6bdadf491b98e0d0b98c56682a: Time spent starting tablet: real 0.035s	user 0.026s	sys 0.008s
14:31:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.360982 30104 heartbeater.cc:344] Connected to a master server at 127.26.209.252:41417
14:31:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.361567 30104 heartbeater.cc:461] Registering TS with master...
14:31:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.362941 30104 heartbeater.cc:507] Master 127.26.209.252:41417 requested a full tablet report, sending...
14:31:05.365 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783521017622, tablet=13b7e17c89d24bf7933f32c1ea7c621f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
14:31:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.365592 30106 heartbeater.cc:344] Connected to a master server at 127.26.209.254:45405
14:31:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.365963 30106 heartbeater.cc:461] Registering TS with master...
14:31:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.366941 30106 heartbeater.cc:507] Master 127.26.209.254:45405 requested a full tablet report, sending...
14:31:05.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.370416 30105 heartbeater.cc:344] Connected to a master server at 127.26.209.253:34165
14:31:05.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.370815 30105 heartbeater.cc:461] Registering TS with master...
14:31:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.371680 30105 heartbeater.cc:507] Master 127.26.209.253:34165 requested a full tablet report, sending...
14:31:05.374 [DEBUG - main] (Connection.java:575) [peer master-127.26.209.254:45405(127.26.209.254:45405)] disconnecting while in state READY
14:31:05.375 [DEBUG - main] (Connection.java:575) [peer 71108c6bdadf491b98e0d0b98c56682a(127.26.209.193:36347)] disconnecting while in state TERMINATED
14:31:05.376 [DEBUG - main] (Connection.java:575) [peer master-127.26.209.253:34165(127.26.209.253:34165)] disconnecting while in state READY
14:31:05.376 [DEBUG - main] (Connection.java:575) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] disconnecting while in state READY
14:31:05.376 [DEBUG - main] (Connection.java:575) [peer b4522c04ae7349348eaf3cf1d3fd07bd(127.26.209.194:44581)] disconnecting while in state TERMINATED
14:31:05.376 [DEBUG - main] (Connection.java:575) [peer master-127.26.209.252:41417(127.26.209.252:41417)] disconnecting while in state READY
14:31:05.378 [DEBUG - kudu-nio-6] (Connection.java:239) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] handling channelInactive
14:31:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.375653 27509 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:31:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.378068 27584 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:31:05.379 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.209.252:41417(127.26.209.252:41417)] handling channelInactive
14:31:05.379 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.209.252:41417(127.26.209.252:41417)] cleaning up while in state READY due to: connection closed
14:31:05.378 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.26.209.254:45405(127.26.209.254:45405)] handling channelInactive
14:31:05.379 [DEBUG - kudu-nio-6] (Connection.java:709) [peer cdc191ea1f6b4308a5a56e280893ec03(127.26.209.195:44447)] cleaning up while in state READY due to: connection closed
14:31:05.380 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.26.209.254:45405(127.26.209.254:45405)] cleaning up while in state READY due to: connection closed
14:31:05.380 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:31:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.381935 27664 ts_manager.cc:194] Re-registered known tserver with Master: 71108c6bdadf491b98e0d0b98c56682a (127.26.209.193:36347)
14:31:05.387 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.209.253:34165(127.26.209.253:34165)] handling channelInactive
14:31:05.388 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.209.253:34165(127.26.209.253:34165)] cleaning up while in state READY due to: connection closed
14:31:05.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.393690 27664 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.209.193:36879
14:31:05.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.395547 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 29969
14:31:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.419221 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 29777
14:31:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.458328 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 29593
14:31:05.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.485935 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 27476
14:31:05.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.520470 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 27546
14:31:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 14:31:05.552819 27463 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskksXv_G/build/tsan/bin/kudu with pid 27621
14:31:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T14:31:06Z chronyd exiting

Time: 64.822

OK (1 test)

