JUnit version 4.13.2
.12:12:33.202 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783512753194
12:12:33.265 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
12:12:33.307 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
12:12:33.323 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskSxpMW8/test-tmp
12:12:33.349 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
12:12:33.353 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskSxpMW8/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
12:12:33.529 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

12:12:33.543 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:12:33.544 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

12:12:33.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.545789  1946 mini_oidc.cc:150] Starting JWKS server
12:12:33.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.546515  1946 webserver.cc:533] Webserver started at http://127.0.0.1:37677/ using document root <none> and password file <none>
12:12:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.546613  1946 mini_oidc.cc:186] Starting OIDC Discovery server
12:12:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.546837  1946 webserver.cc:533] Webserver started at http://127.0.0.1:39165/ using document root <none> and password file <none>
12:12:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:12:33Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
12:12:33.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:12:33Z Disabled control of system clock
12:12:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.558462  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:33.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:33.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/wal
12:12:33.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data
12:12:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/logs
12:12:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data/info.pb
12:12:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:12:33.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:33.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:33.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.190:40253
12:12:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.190
12:12:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37677/jwks.json with env {}
12:12:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.638649  1958 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.638895  1958 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:33.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.638914  1958 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:33.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.640528  1958 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:12:33.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.640569  1958 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.640583  1958 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:12:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.640595  1958 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:12:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.640607  1958 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:12:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.640619  1958 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37677/jwks.json
12:12:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.642359  1958 master_runner.cc:387] Master server non-default flags:
12:12:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data
12:12:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/wal
12:12:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:33.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.190:40253
12:12:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37677/jwks.json
12:12:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data/info.pb
12:12:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.190
12:12:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1958
12:12:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/logs
12:12:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:12:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.642582  1958 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.642887  1958 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.645746  1964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.645787  1963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.645836  1966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.646018  1958 server_base.cc:1061] running on GCE node
12:12:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.646183  1958 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.646389  1958 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.647523  1958 hybrid_clock.cc:648] HybridClock initialized: now 1783512753647494 us; error 44 us; skew 500 ppm
12:12:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.648695  1958 webserver.cc:533] Webserver started at http://127.1.230.190:43885/ using document root <none> and password file <none>
12:12:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.648862  1958 fs_manager.cc:362] Metadata directory not provided
12:12:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.648898  1958 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.648969  1958 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.649741  1958 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data/instance:
12:12:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3abba7d82364ddd8cd110d64792c26f"
12:12:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.650004  1958 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/wal/instance:
12:12:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3abba7d82364ddd8cd110d64792c26f"
12:12:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.653052  1958 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.000s	sys 0.002s
12:12:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.653899  1972 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.654060  1958 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
12:12:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.654114  1958 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/wal
12:12:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3abba7d82364ddd8cd110d64792c26f"
12:12:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.654167  1958 fs_report.cc:389] FS layout report
12:12:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/wal
12:12:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/wal
12:12:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data/data
12:12:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:33.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:33.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:33.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:33.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:33.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:33.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.670064  1958 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:33.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.670318  1958 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:33.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.670401  1958 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:33.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.674512  1958 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.190:40253
12:12:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.674541  2024 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.190:40253 every 8 connection(s)
12:12:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.674953  1958 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/data/info.pb
12:12:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.675524  2025 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.676384  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 1958
12:12:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.676494  1946 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-0/wal/instance
12:12:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.676836  2025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.677354  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/wal
12:12:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data
12:12:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/logs
12:12:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data/info.pb
12:12:33.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:33.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:33.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:33.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.189:40567
12:12:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.189
12:12:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37677/jwks.json with env {}
12:12:33.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.683987  2025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.684680  1975 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.189:40567: connect: Connection refused (error 111)
12:12:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.691102  2025 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.189:40567: Network error: Client connection negotiation failed: client connection to 127.1.230.189:40567: connect: Connection refused (error 111)
12:12:33.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.740351  2025 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } attempt: 1
12:12:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.741262  2025 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.189:40567: Network error: Client connection negotiation failed: client connection to 127.1.230.189:40567: connect: Connection refused (error 111)
12:12:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.757776  2029 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.757961  2029 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.757989  2029 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.759899  2029 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:12:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.759975  2029 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.759999  2029 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:12:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.760020  2029 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:12:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.760040  2029 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:12:33.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.760058  2029 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37677/jwks.json
12:12:33.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.762395  2029 master_runner.cc:387] Master server non-default flags:
12:12:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data
12:12:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/wal
12:12:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.189:40567
12:12:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37677/jwks.json
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data/info.pb
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.189
12:12:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.2029
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/logs
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.762692  2029 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.762936  2029 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.765484  2038 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.765678  2036 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.765832  2035 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.769498  2029 server_base.cc:1061] running on GCE node
12:12:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.769778  2029 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.770087  2029 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:33.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.771238  2029 hybrid_clock.cc:648] HybridClock initialized: now 1783512753771218 us; error 39 us; skew 500 ppm
12:12:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.772567  2029 webserver.cc:533] Webserver started at http://127.1.230.189:38485/ using document root <none> and password file <none>
12:12:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.772830  2029 fs_manager.cc:362] Metadata directory not provided
12:12:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.772891  2029 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.772997  2029 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.774020  2029 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data/instance:
12:12:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0630f93240347729a6fe34f25bfdc31"
12:12:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.774413  2029 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/wal/instance:
12:12:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0630f93240347729a6fe34f25bfdc31"
12:12:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.775934  2029 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.003s	sys 0.000s
12:12:33.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.776710  2044 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.776863  2029 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
12:12:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.776916  2029 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/wal
12:12:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0630f93240347729a6fe34f25bfdc31"
12:12:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.776968  2029 fs_report.cc:389] FS layout report
12:12:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/wal
12:12:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/wal
12:12:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data/data
12:12:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:33.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:33.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:33.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:33.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.809440  2025 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } attempt: 2
12:12:33.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.809589  2029 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:33.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.809875  2029 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:33.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.810025  2029 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.810310  2025 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.189:40567: Network error: Client connection negotiation failed: client connection to 127.1.230.189:40567: connect: Connection refused (error 111)
12:12:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.814754  2029 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.189:40567
12:12:33.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.814828  2096 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.189:40567 every 8 connection(s)
12:12:33.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.815056  2029 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/data/info.pb
12:12:33.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.815626  2097 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:33.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.817054  2097 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.822333  2097 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.822737  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 2029
12:12:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.822824  1946 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-1/wal/instance
12:12:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.823372  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/wal
12:12:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data
12:12:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/logs
12:12:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data/info.pb
12:12:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
12:12:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.188:43411
12:12:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.188
12:12:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37677/jwks.json with env {}
12:12:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.828256  2097 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.828733  2048 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.188:43411: connect: Connection refused (error 111)
12:12:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.831100  2097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.188:43411: Network error: Client connection negotiation failed: client connection to 127.1.230.188:43411: connect: Connection refused (error 111)
12:12:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.880307  2097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } attempt: 1
12:12:33.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.881290  2097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.188:43411: Network error: Client connection negotiation failed: client connection to 127.1.230.188:43411: connect: Connection refused (error 111)
12:12:33.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.901471  2025 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } attempt: 3
12:12:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.904794  2025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.905467  2025 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.188:43411: Network error: Client connection negotiation failed: client connection to 127.1.230.188:43411: connect: Connection refused (error 111)
12:12:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.914390  2101 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.914625  2101 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.914659  2101 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.916600  2101 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
12:12:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.916690  2101 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.916714  2101 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
12:12:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.916733  2101 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
12:12:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.916754  2101 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
12:12:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.916774  2101 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37677/jwks.json
12:12:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.918931  2101 master_runner.cc:387] Master server non-default flags:
12:12:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data
12:12:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/wal
12:12:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
12:12:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
12:12:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.188:43411
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37677/jwks.json
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data/info.pb
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.188
12:12:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.2101
12:12:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/logs
12:12:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
12:12:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.919212  2101 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.919498  2101 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.922216  2107 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.922454  2108 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.922778  2110 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.923460  2101 server_base.cc:1061] running on GCE node
12:12:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.923655  2101 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.923853  2101 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.927094  2101 hybrid_clock.cc:648] HybridClock initialized: now 1783512753927067 us; error 33 us; skew 500 ppm
12:12:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.928473  2101 webserver.cc:533] Webserver started at http://127.1.230.188:45041/ using document root <none> and password file <none>
12:12:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.928682  2101 fs_manager.cc:362] Metadata directory not provided
12:12:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.928731  2101 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.928823  2101 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.929896  2101 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data/instance:
12:12:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c6a373b1d9d4fd390dfafb179d3bb37"
12:12:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.930337  2101 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/wal/instance:
12:12:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c6a373b1d9d4fd390dfafb179d3bb37"
12:12:33.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.931854  2101 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.003s
12:12:33.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.932629  2116 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:33.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.932808  2101 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.932883  2101 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/wal
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c6a373b1d9d4fd390dfafb179d3bb37"
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:33 on dist-test-slave-dp7z"
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.932928  2101 fs_report.cc:389] FS layout report
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/wal
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/wal
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data/data
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:33.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.936611  2025 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } attempt: 1
12:12:33.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.937402  2025 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.188:43411: Network error: Client connection negotiation failed: client connection to 127.1.230.188:43411: connect: Connection refused (error 111)
12:12:33.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.944959  2101 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:33.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.945331  2101 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:33.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.945472  2101 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:33.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.949433  2097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } attempt: 2
12:12:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.950294  2097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.230.188:43411: Network error: Client connection negotiation failed: client connection to 127.1.230.188:43411: connect: Connection refused (error 111)
12:12:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.950832  2101 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.188:43411
12:12:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.950866  2168 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.188:43411 every 8 connection(s)
12:12:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.951279  2101 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/data/info.pb
12:12:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.951828  2169 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.953167  2169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.958335  2169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.958914  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 2101
12:12:33.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.959023  1946 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/master-2/wal/instance
12:12:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.961638  2169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } has no permanent_uuid. Determining permanent_uuid...
12:12:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.966464  2169 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37: Bootstrap starting.
12:12:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.967196  2169 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37: Neither blocks nor log segments found. Creating new log.
12:12:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.967487  2169 log.cc:826] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37: Log is configured to *not* fsync() on all Append() calls
12:12:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.968919  2169 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37: No bootstrap required, opened a new log
12:12:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.971203  2169 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.971315  2169 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.971338  2169 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2c6a373b1d9d4fd390dfafb179d3bb37, State: Initialized, Role: FOLLOWER
12:12:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.971434  2169 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.971657  2169 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: configured and running, proceeding with master startup.
12:12:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.971684  2177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.971773  2177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: This master's current role is: FOLLOWER
12:12:33.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.973490  2188 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:12:33.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:33.973548  2188 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:12:33.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.973677  2101 master_runner.cc:187] Error getting master registration for 127.1.230.190:40253: OK, instance_id { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" instance_seqno: 1783512753673488 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:12:33.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:33.974949  2101 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:12:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.012565  2025 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } attempt: 2
12:12:34.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.016623  2025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: Bootstrap starting.
12:12:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.017421  2025 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: Neither blocks nor log segments found. Creating new log.
12:12:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.017730  2025 log.cc:826] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: Log is configured to *not* fsync() on all Append() calls
12:12:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.018519  2025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: No bootstrap required, opened a new log
12:12:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.020692  2025 raft_consensus.cc:359] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.020815  2025 raft_consensus.cc:385] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.020846  2025 raft_consensus.cc:740] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3abba7d82364ddd8cd110d64792c26f, State: Initialized, Role: FOLLOWER
12:12:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.020936  2025 consensus_queue.cc:260] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.021220  2025 sys_catalog.cc:565] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [sys.catalog]: configured and running, proceeding with master startup.
12:12:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.021837  2191 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.022176  2191 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [sys.catalog]: This master's current role is: FOLLOWER
12:12:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.023180  2202 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:12:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.023466  2202 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:12:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.023880  1958 master_runner.cc:187] Error getting master registration for 127.1.230.189:40567: OK, instance_id { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" instance_seqno: 1783512753813659 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
12:12:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.025815  1958 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:12:34.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.041471  2097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } attempt: 3
12:12:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.046375  2097 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: Bootstrap starting.
12:12:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.047096  2097 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: Neither blocks nor log segments found. Creating new log.
12:12:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.047363  2097 log.cc:826] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: Log is configured to *not* fsync() on all Append() calls
12:12:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.048045  2097 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: No bootstrap required, opened a new log
12:12:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.049888  2097 raft_consensus.cc:359] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.050015  2097 raft_consensus.cc:385] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.050047  2097 raft_consensus.cc:740] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0630f93240347729a6fe34f25bfdc31, State: Initialized, Role: FOLLOWER
12:12:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.050136  2097 consensus_queue.cc:260] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.050393  2097 sys_catalog.cc:565] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [sys.catalog]: configured and running, proceeding with master startup.
12:12:34.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.050873  2205 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.051155  2205 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [sys.catalog]: This master's current role is: FOLLOWER
12:12:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.052328  2216 catalog_manager.cc:1594] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
12:12:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.052631  2216 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
12:12:34.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.054432  2029 master_runner.cc:264] No leader master found from master b0630f93240347729a6fe34f25bfdc31
12:12:34.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.054486  2029 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
12:12:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.060647  2050 tablet.cc:2406] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:34.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.069772  2177 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:34.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.069882  2177 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:34.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.070093  2177 leader_election.cc:290] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3abba7d82364ddd8cd110d64792c26f (127.1.230.190:40253), b0630f93240347729a6fe34f25bfdc31 (127.1.230.189:40567)
12:12:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071079  1999 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3abba7d82364ddd8cd110d64792c26f" is_pre_election: true
12:12:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071164  2071 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b0630f93240347729a6fe34f25bfdc31" is_pre_election: true
12:12:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071224  1999 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2c6a373b1d9d4fd390dfafb179d3bb37 in term 0.
12:12:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071291  2071 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2c6a373b1d9d4fd390dfafb179d3bb37 in term 0.
12:12:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071477  2119 leader_election.cc:304] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2c6a373b1d9d4fd390dfafb179d3bb37, b0630f93240347729a6fe34f25bfdc31; no voters: 
12:12:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071643  2177 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Leader pre-election won for term 1
12:12:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071682  2177 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:12:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.071704  2177 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 0 FOLLOWER]: Advancing to term 1
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.072539  2177 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.072667  2177 leader_election.cc:290] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [CANDIDATE]: Term 1 election: Requested vote from peers a3abba7d82364ddd8cd110d64792c26f (127.1.230.190:40253), b0630f93240347729a6fe34f25bfdc31 (127.1.230.189:40567)
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.072903  1999 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3abba7d82364ddd8cd110d64792c26f"
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.072968  1999 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [term 0 FOLLOWER]: Advancing to term 1
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.073715  1999 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2c6a373b1d9d4fd390dfafb179d3bb37 in term 1.
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.074008  2117 leader_election.cc:304] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2c6a373b1d9d4fd390dfafb179d3bb37, a3abba7d82364ddd8cd110d64792c26f; no voters: 
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.074090  2071 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b0630f93240347729a6fe34f25bfdc31"
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.074158  2071 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [term 0 FOLLOWER]: Advancing to term 1
12:12:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.074947  2071 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2c6a373b1d9d4fd390dfafb179d3bb37 in term 1.
12:12:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.075155  2177 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 1 FOLLOWER]: Leader election won for term 1
12:12:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.075310  2177 raft_consensus.cc:697] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [term 1 LEADER]: Becoming Leader. State: Replica: 2c6a373b1d9d4fd390dfafb179d3bb37, State: Running, Role: LEADER
12:12:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.075387  2177 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } }
12:12:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.075707  2177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2c6a373b1d9d4fd390dfafb179d3bb37. Latest consensus state: current_term: 1 leader_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.075775  2177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: This master's current role is: LEADER
12:12:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.080848  2220 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
12:12:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.081427  2220 catalog_manager.cc:1520] Initializing Kudu cluster ID...
12:12:34.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.089118  2071 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [term 1 FOLLOWER]: Refusing update from remote peer 2c6a373b1d9d4fd390dfafb179d3bb37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:34.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.090302  2177 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:12:34.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.091434  2205 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2c6a373b1d9d4fd390dfafb179d3bb37. Latest consensus state: current_term: 1 leader_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.091512  2205 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [sys.catalog]: This master's current role is: FOLLOWER
12:12:34.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.093523  2205 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.093595  2205 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31 [sys.catalog]: This master's current role is: FOLLOWER
12:12:34.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.094606  2177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.094703  2177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: This master's current role is: LEADER
12:12:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.094769  1999 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [term 1 FOLLOWER]: Refusing update from remote peer 2c6a373b1d9d4fd390dfafb179d3bb37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.095201  2177 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:12:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.095335  2222 mvcc.cc:204] Tried to move back new op lower bound from 7305268240746622976 to 7305268240693346304. Current Snapshot: MvccSnapshot[applied={T|T < 7305268240746622976}]
12:12:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.095430  2221 mvcc.cc:204] Tried to move back new op lower bound from 7305268240746622976 to 7305268240693346304. Current Snapshot: MvccSnapshot[applied={T|T < 7305268240746622976}]
12:12:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.096354  2191 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2c6a373b1d9d4fd390dfafb179d3bb37. Latest consensus state: current_term: 1 leader_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.096432  2191 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [sys.catalog]: This master's current role is: FOLLOWER
12:12:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.096468  2191 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.096504  2191 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f [sys.catalog]: This master's current role is: FOLLOWER
12:12:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.099123  2220 catalog_manager.cc:1383] Generated new cluster ID: 012e9543fd7d40afb275b3c9a9744bfc
12:12:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.099177  2220 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
12:12:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.101519  2227 mvcc.cc:204] Tried to move back new op lower bound from 7305268240746622976 to 7305268240693346304. Current Snapshot: MvccSnapshot[applied={T|T < 7305268240746622976}]
12:12:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.127857  2177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3abba7d82364ddd8cd110d64792c26f" member_type: VOTER last_known_addr { host: "127.1.230.190" port: 40253 } } peers { permanent_uuid: "b0630f93240347729a6fe34f25bfdc31" member_type: VOTER last_known_addr { host: "127.1.230.189" port: 40567 } } peers { permanent_uuid: "2c6a373b1d9d4fd390dfafb179d3bb37" member_type: VOTER last_known_addr { host: "127.1.230.188" port: 43411 } } }
12:12:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.127978  2177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37 [sys.catalog]: This master's current role is: LEADER
12:12:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.131548  2220 catalog_manager.cc:1406] Generated new certificate authority record
12:12:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.133344  2220 catalog_manager.cc:1540] Loading token signing keys...
12:12:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.146361  2220 catalog_manager.cc:6070] T 00000000000000000000000000000000 P 2c6a373b1d9d4fd390dfafb179d3bb37: Generated new TSK 0
12:12:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.146915  2220 catalog_manager.cc:1550] Initializing in-progress tserver states...
12:12:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.152004  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:12:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:0
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:34.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.240466  2231 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:34.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.240649  2231 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:34.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.240679  2231 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:34.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.242316  2231 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:34.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.242400  2231 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.129
12:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.244035  2231 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:0
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.2231
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.244285  2231 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.244536  2231 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:34.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.249435  2237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.249641  2239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.249843  2236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.250494  2231 server_base.cc:1061] running on GCE node
12:12:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.250656  2231 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.250918  2231 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.252254  2231 hybrid_clock.cc:648] HybridClock initialized: now 1783512754252214 us; error 54 us; skew 500 ppm
12:12:34.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.253340  2231 webserver.cc:533] Webserver started at http://127.1.230.129:34273/ using document root <none> and password file <none>
12:12:34.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.253587  2231 fs_manager.cc:362] Metadata directory not provided
12:12:34.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.253676  2231 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.253793  2231 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.254908  2231 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/instance:
12:12:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "968c62486e7148c4978383fb424be31b"
12:12:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.255252  2231 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal/instance:
12:12:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "968c62486e7148c4978383fb424be31b"
12:12:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.256846  2231 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.001s	sys 0.001s
12:12:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.257543  2245 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.257695  2231 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.257745  2231 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "968c62486e7148c4978383fb424be31b"
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.257812  2231 fs_report.cc:389] FS layout report
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/data
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.276813  2231 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.277139  2231 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.277263  2231 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.277485  2231 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.277868  2231 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:12:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.277905  2231 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:12:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.278008  2231 ts_tablet_manager.cc:616] Registered 0 tablets
12:12:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.278033  2231 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:12:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.284418  2231 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.129:39309
12:12:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.284459  2358 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.129:39309 every 8 connection(s)
12:12:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.284747  2231 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:34.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.286903  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 2231
12:12:34.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.287001  1946 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal/instance
12:12:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.288985  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:12:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:0
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.291127  2360 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.291217  2360 heartbeater.cc:461] Registering TS with master...
12:12:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.291433  2360 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.291872  1988 ts_manager.cc:194] Registered new tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:34.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.293191  2361 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:34.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.293247  2361 heartbeater.cc:461] Registering TS with master...
12:12:34.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.293349  2361 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:34.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.293689  2061 ts_manager.cc:194] Registered new tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.296157  2359 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.296216  2359 heartbeater.cc:461] Registering TS with master...
12:12:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.296283  2359 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.296548  2133 ts_manager.cc:194] Registered new tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:34.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.297125  2133 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.129:41415
12:12:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.369139  2366 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.369367  2366 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.369395  2366 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:34.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.371078  2366 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:34.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.371151  2366 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.130
12:12:34.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.372768  2366 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:0
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.2366
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:34.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:12:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:12:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.373015  2366 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.373311  2366 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.375964  2374 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.375990  2371 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.376165  2372 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.376405  2366 server_base.cc:1061] running on GCE node
12:12:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.376552  2366 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.376770  2366 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:34.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.377915  2366 hybrid_clock.cc:648] HybridClock initialized: now 1783512754377885 us; error 46 us; skew 500 ppm
12:12:34.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.379091  2366 webserver.cc:533] Webserver started at http://127.1.230.130:46119/ using document root <none> and password file <none>
12:12:34.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.379297  2366 fs_manager.cc:362] Metadata directory not provided
12:12:34.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.379370  2366 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:34.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.379478  2366 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:34.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.380337  2366 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/instance:
12:12:34.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:34.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.380686  2366 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal/instance:
12:12:34.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:34.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.382021  2366 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.003s
12:12:34.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.382750  2380 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.382905  2366 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.382954  2366 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.383002  2366 fs_report.cc:389] FS layout report
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/data
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:34.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:34.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.396689  2366 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.396935  2366 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.397013  2366 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.397198  2366 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.397533  2366 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.397567  2366 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.397586  2366 ts_tablet_manager.cc:616] Registered 0 tablets
12:12:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.397598  2366 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:12:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.404369  2366 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.130:42273
12:12:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.404410  2493 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.130:42273 every 8 connection(s)
12:12:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.404688  2366 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.406826  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 2366
12:12:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.406916  1946 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal/instance
12:12:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.408789  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:0
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.410434  2494 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.410492  2495 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.410516  2494 heartbeater.cc:461] Registering TS with master...
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.410739  2496 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.410805  2496 heartbeater.cc:461] Registering TS with master...
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.410935  2496 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.410526  2495 heartbeater.cc:461] Registering TS with master...
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.411180  2495 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.411367  2061 ts_manager.cc:194] Registered new tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.411445  1988 ts_manager.cc:194] Registered new tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.411697  2494 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.411914  2133 ts_manager.cc:194] Registered new tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.412235  2133 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.130:38017
12:12:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.487488  2501 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.487694  2501 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.487720  2501 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.489326  2501 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.489408  2501 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.131
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.491039  2501 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:0
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.2501
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.491289  2501 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.491540  2501 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.494298  2507 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.494324  2509 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:34.494347  2506 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.494508  2501 server_base.cc:1061] running on GCE node
12:12:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.494858  2501 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.495008  2501 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.496120  2501 hybrid_clock.cc:648] HybridClock initialized: now 1783512754496096 us; error 43 us; skew 500 ppm
12:12:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.497048  2501 webserver.cc:533] Webserver started at http://127.1.230.131:33199/ using document root <none> and password file <none>
12:12:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.497189  2501 fs_manager.cc:362] Metadata directory not provided
12:12:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.497221  2501 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.497289  2501 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
12:12:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.498018  2501 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/instance:
12:12:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57953137991b435ea05b795d05ae73c9"
12:12:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.498334  2501 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal/instance:
12:12:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57953137991b435ea05b795d05ae73c9"
12:12:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.499828  2501 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.003s
12:12:34.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.500584  2515 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:34.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.500765  2501 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.001s
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.500828  2501 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57953137991b435ea05b795d05ae73c9"
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.500877  2501 fs_report.cc:389] FS layout report
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/data
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512182  2501 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512391  2501 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512468  2501 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512600  2501 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512809  2501 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
12:12:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512836  2501 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
12:12:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512856  2501 ts_tablet_manager.cc:616] Registered 0 tablets
12:12:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.512868  2501 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
12:12:34.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.518965  2501 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.131:38541
12:12:34.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.519007  2628 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.131:38541 every 8 connection(s)
12:12:34.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.519274  2501 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:34.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524255  2629 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:34.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524271  2630 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:34.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524353  2629 heartbeater.cc:461] Registering TS with master...
12:12:34.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524502  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 2501
12:12:34.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524559  1946 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal/instance
12:12:34.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524544  2629 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:34.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524350  2630 heartbeater.cc:461] Registering TS with master...
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.524791  2630 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.525122  1988 ts_manager.cc:194] Registered new tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.525136  2133 ts_manager.cc:194] Registered new tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.525388  2631 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.525425  2631 heartbeater.cc:461] Registering TS with master...
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.525498  2631 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.525606  2133 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.131:48647
12:12:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.525736  2061 ts_manager.cc:194] Registered new tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:34.526721  1946 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
12:12:34.527 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
12:12:34.529 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

12:12:34.537 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.1.230.190"
      port: 40253
    }
    bound_http_address {
      host: "127.1.230.190"
      port: 43885
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.1.230.189"
      port: 40567
    }
    bound_http_address {
      host: "127.1.230.189"
      port: 38485
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.1.230.188"
      port: 43411
    }
    bound_http_address {
      host: "127.1.230.188"
      port: 45041
    }
  }
}

12:12:34.540 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

12:12:34.542 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.1.230.129"
      port: 39309
    }
    bound_http_address {
      host: "127.1.230.129"
      port: 34273
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.1.230.130"
      port: 42273
    }
    bound_http_address {
      host: "127.1.230.130"
      port: 46119
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.1.230.131"
      port: 38541
    }
    bound_http_address {
      host: "127.1.230.131"
      port: 33199
    }
  }
}

12:12:34.542 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
12:12:34.720 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
12:12:34.765 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
12:12:34.913 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.230.190:40253(127.1.230.190:40253)] connecting to peer
12:12:34.974 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.230.189:40567(127.1.230.189:40567)] connecting to peer
12:12:34.975 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.230.188:43411(127.1.230.188:43411)] connecting to peer
12:12:34.998 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.1.230.188:43411(127.1.230.188:43411)] Successfully connected to peer
12:12:34.998 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.230.189:40567(127.1.230.189:40567)] Successfully connected to peer
12:12:34.998 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.230.190:40253(127.1.230.190:40253)] Successfully connected to peer
12:12:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.024158  2202 catalog_manager.cc:1295] Loaded cluster ID: 012e9543fd7d40afb275b3c9a9744bfc
12:12:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.024240  2202 catalog_manager.cc:1588] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: loading cluster ID for follower catalog manager: success
12:12:35.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.025542  2202 catalog_manager.cc:1610] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: acquiring CA information for follower catalog manager: success
12:12:35.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.025867  2202 catalog_manager.cc:1638] T 00000000000000000000000000000000 P a3abba7d82364ddd8cd110d64792c26f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:12:35.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.053280  2216 catalog_manager.cc:1295] Loaded cluster ID: 012e9543fd7d40afb275b3c9a9744bfc
12:12:35.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.053360  2216 catalog_manager.cc:1588] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: loading cluster ID for follower catalog manager: success
12:12:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.054768  2216 catalog_manager.cc:1610] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: acquiring CA information for follower catalog manager: success
12:12:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.055058  2216 catalog_manager.cc:1638] T 00000000000000000000000000000000 P b0630f93240347729a6fe34f25bfdc31: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
12:12:35.105 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.230.188
12:12:35.105 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.230.190
12:12:35.107 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.230.189
12:12:35.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.297734  2359 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:35.316 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x12538189, L:/127.0.0.1:59260 - R:/127.1.230.190:40253] using SASL/PLAIN
12:12:35.316 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x4a0d158f, L:/127.0.0.1:46722 - R:/127.1.230.188:43411] using SASL/PLAIN
12:12:35.317 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x56c12e9a, L:/127.0.0.1:52554 - R:/127.1.230.189:40567] using SASL/PLAIN
12:12:35.354 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
12:12:35.354 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=480), Traces: [0ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.230.190:40253, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.230.189:40567, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.230.188:43411, [451ms] Sub RPC ConnectToMaster: received response from server master-127.1.230.190:40253: OK, [451ms] Sub RPC ConnectToMaster: received response from server master-127.1.230.189:40567: OK, [452ms] Sub RPC ConnectToMaster: received response from server master-127.1.230.188:43411: OK, deferred=null) after lookup
12:12:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.398138  2132 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46722:
12:12:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859"
12:12:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
12:12:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
12:12:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
12:12:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
12:12:35.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
12:12:35.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
12:12:35.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
12:12:35.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.398550  2132 catalog_manager.cc:7077] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
12:12:35.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.408939  2428 tablet_service.cc:1511] Processing CreateTablet for tablet c68a9df270ca4b13b47fb31a1b9431a7 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859 [id=209560dee3494475af0a85abbcf71c7c]), partition=RANGE (key) PARTITION UNBOUNDED
12:12:35.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.409251  2428 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c68a9df270ca4b13b47fb31a1b9431a7. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:35.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.412787  2494 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.415211  2658 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap starting.
12:12:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.415833  2658 tablet_bootstrap.cc:654] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Neither blocks nor log segments found. Creating new log.
12:12:35.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.418645  2658 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Log is configured to *not* fsync() on all Append() calls
12:12:35.425 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 209560dee3494475af0a85abbcf71c7c
12:12:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.423563  2563 tablet_service.cc:1511] Processing CreateTablet for tablet c68a9df270ca4b13b47fb31a1b9431a7 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859 [id=209560dee3494475af0a85abbcf71c7c]), partition=RANGE (key) PARTITION UNBOUNDED
12:12:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.423872  2563 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c68a9df270ca4b13b47fb31a1b9431a7. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.424381  2293 tablet_service.cc:1511] Processing CreateTablet for tablet c68a9df270ca4b13b47fb31a1b9431a7 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859 [id=209560dee3494475af0a85abbcf71c7c]), partition=RANGE (key) PARTITION UNBOUNDED
12:12:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.424602  2293 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c68a9df270ca4b13b47fb31a1b9431a7. 1 dirs total, 0 dirs full, 0 dirs failed
12:12:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.426849  2662 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap starting.
12:12:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.427265  2658 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: No bootstrap required, opened a new log
12:12:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.427340  2658 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent bootstrapping tablet: real 0.012s	user 0.001s	sys 0.000s
12:12:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.427321  2663 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap starting.
12:12:35.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.427938  2663 tablet_bootstrap.cc:654] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Neither blocks nor log segments found. Creating new log.
12:12:35.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.428376  2663 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Log is configured to *not* fsync() on all Append() calls
12:12:35.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.428632  2662 tablet_bootstrap.cc:654] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Neither blocks nor log segments found. Creating new log.
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.428961  2658 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.429050  2662 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Log is configured to *not* fsync() on all Append() calls
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.429116  2663 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: No bootstrap required, opened a new log
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.429109  2658 raft_consensus.cc:385] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.429148  2658 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010544b667d64cc6b3fa372f3d439d95, State: Initialized, Role: FOLLOWER
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.429168  2663 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent bootstrapping tablet: real 0.002s	user 0.001s	sys 0.000s
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.429235  2658 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.429463  2658 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
12:12:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.431735  2663 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.431869  2663 raft_consensus.cc:385] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.431905  2663 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 968c62486e7148c4978383fb424be31b, State: Initialized, Role: FOLLOWER
12:12:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.431996  2663 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.432116  2662 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: No bootstrap required, opened a new log
12:12:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.432185  2662 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent bootstrapping tablet: real 0.005s	user 0.001s	sys 0.000s
12:12:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.432228  2663 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent starting tablet: real 0.003s	user 0.003s	sys 0.000s
12:12:35.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.433712  2662 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.434005  2662 raft_consensus.cc:385] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
12:12:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.434190  2662 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Initialized, Role: FOLLOWER
12:12:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.434311  2662 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.434518  2662 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
12:12:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.434571  2629 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.520345  2632 tablet.cc:2406] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.536655  2362 tablet.cc:2406] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.579994  2668 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.580111  2668 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.580389  2668 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:35.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.583564  2313 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b" is_pre_election: true
12:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.583765  2313 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 0.
12:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.583760  2448 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95" is_pre_election: true
12:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.583909  2448 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 0.
12:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.584034  2518 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b; no voters: 
12:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.584204  2668 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Leader pre-election won for term 1
12:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.584259  2668 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
12:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.584285  2668 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 0 FOLLOWER]: Advancing to term 1
12:12:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.585436  2668 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.585614  2668 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 1 election: Requested vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.585778  2313 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b"
12:12:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.585821  2448 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.585897  2448 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 0 FOLLOWER]: Advancing to term 1
12:12:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.585886  2313 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 0 FOLLOWER]: Advancing to term 1
12:12:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.586472  2448 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 1.
12:12:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.586688  2313 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 1.
12:12:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.586699  2516 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9; no voters: 
12:12:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.586866  2668 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 1 FOLLOWER]: Leader election won for term 1
12:12:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.587023  2668 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 1 LEADER]: Becoming Leader. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Running, Role: LEADER
12:12:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.587128  2668 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:35.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.587891  2132 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 57953137991b435ea05b795d05ae73c9 (127.1.230.131). New cstate: current_term: 1 leader_uuid: "57953137991b435ea05b795d05ae73c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: HEALTHY } } }
12:12:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:35.655941  2497 tablet.cc:2406] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Can't schedule compaction. Clean time has not been advanced past its initial value.
12:12:35.660 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 209560dee3494475af0a85abbcf71c7c
12:12:35.675 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:35.681 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:35.682 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [6ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=null) after lookup
12:12:35.691 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:35.693 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] Successfully connected to peer
12:12:35.761 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x405d0027, L:/127.0.0.1:46202 - R:/127.1.230.131:38541] using TOKEN/null
12:12:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.764353  2448 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 1 FOLLOWER]: Refusing update from remote peer 57953137991b435ea05b795d05ae73c9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.764380  2313 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 1 FOLLOWER]: Refusing update from remote peer 57953137991b435ea05b795d05ae73c9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
12:12:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.764894  2668 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:12:35.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.765300  2668 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
12:12:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.769805  2676 mvcc.cc:204] Tried to move back new op lower bound from 7305268247608037376 to 7305268246885441536. Current Snapshot: MvccSnapshot[applied={T|T < 7305268247608037376 or (T in {7305268247608037376})}]
12:12:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.771152  2678 mvcc.cc:204] Tried to move back new op lower bound from 7305268247608037376 to 7305268246885441536. Current Snapshot: MvccSnapshot[applied={T|T < 7305268247608037376}]
12:12:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.772310  2677 mvcc.cc:204] Tried to move back new op lower bound from 7305268247608037376 to 7305268246885441536. Current Snapshot: MvccSnapshot[applied={T|T < 7305268247608037376}]
12:12:35.842 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:35.843 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.131:38541
12:12:35.845 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:35.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:35.845523  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 2501
12:12:35.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:35.853 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] handling channelInactive
12:12:35.854 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state READY due to: connection closed
12:12:35.854 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: connection closed
12:12:35.878 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:35.879 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:35.879 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [3ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2056618663(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@4d214d27 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ab5496d -> wakeup thread Time-limited test)) after lookup
12:12:35.880 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:35.883 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.883 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.883 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.897 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:35.898 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:35.899 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [3ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [29ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [32ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [32ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2056618663(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@4d214d27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ab5496d -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446))) after lookup
12:12:35.899 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:35.901 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.901 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.901 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.919 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:35.919 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:35.920 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=68), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [3ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [29ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [32ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [32ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [48ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [50ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [51ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2056618663(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@4d214d27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ab5496d -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853))) after lookup
12:12:35.921 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:35.923 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.923 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.923 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.938 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:35.938 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:35.939 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [3ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [29ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [32ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [32ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [48ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [50ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [51ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [69ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [72ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [72ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [86ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2056618663(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@4d214d27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ab5496d -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291))) after lookup
12:12:35.939 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:35.941 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.941 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:35.941 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:36.038 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:36.039 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:36.039 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=188), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [3ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [29ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [32ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [32ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [48ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [50ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [51ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [69ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [72ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [72ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [86ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [88ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [90ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [90ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [187ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2056618663(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@4d214d27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291) -> (continuation of Deferred@1767804751 after retry RPC@1855205450), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ab5496d -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291) -> (continuation of Deferred@1767804751 after retry RPC@1855205450))) after lookup
12:12:36.040 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:36.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:36.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:36.041 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:36.438 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:36.438 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:36.439 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=587), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [3ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [29ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [32ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [32ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [48ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [50ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [51ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [69ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [72ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [72ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [86ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [88ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [90ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [90ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [187ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [188ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [190ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [190ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [584ms] refreshing cache from master, [584ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [587ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2056618663(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@4d214d27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291) -> (continuation of Deferred@1767804751 after retry RPC@1855205450) -> (continuation of Deferred@1860711059 after retry RPC@556882276), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ab5496d -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291) -> (continuation of Deferred@1767804751 after retry RPC@1855205450) -> (continuation of Deferred@1860711059 after retry RPC@556882276))) after lookup
12:12:36.440 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:36.441 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:36.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:36.441 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:37.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.296818  2690 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:12:37.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.296914  2690 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:37.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.297204  2690 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:37.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:37.297713  2249 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.297741  2693 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:12:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.297816  2693 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:37.298153  2249 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.298584  2693 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:37.298836  2384 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:37.299401  2384 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.300625  2448 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95" is_pre_election: true
12:12:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.300724  2448 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 1.
12:12:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.300884  2246 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 57953137991b435ea05b795d05ae73c9
12:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.301002  2690 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 1 FOLLOWER]: Leader pre-election won for term 2
12:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.301046  2690 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 1 FOLLOWER]: Starting leader election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.301062  2690 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 1 FOLLOWER]: Advancing to term 2
12:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.301394  2313 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b" is_pre_election: true
12:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.301745  2690 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.301873  2313 raft_consensus.cc:2393] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 010544b667d64cc6b3fa372f3d439d95 in current term 2: Already voted for candidate 968c62486e7148c4978383fb424be31b in this term.
12:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.301899  2690 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 2 election: Requested vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.302023  2448 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.302083  2448 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 1 FOLLOWER]: Advancing to term 2
12:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.302067  2383 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95; no voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b
12:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.302742  2448 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 2.
12:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.302824  2693 raft_consensus.cc:2749] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
12:12:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.302922  2246 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 
12:12:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.302992  2690 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Leader election won for term 2
12:12:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.303107  2690 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 LEADER]: Becoming Leader. State: Replica: 968c62486e7148c4978383fb424be31b, State: Running, Role: LEADER
12:12:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.303181  2690 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:37.303357  2249 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.303797  2132 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b reported cstate change: term changed from 1 to 2, leader changed from 57953137991b435ea05b795d05ae73c9 (127.1.230.131) to 968c62486e7148c4978383fb424be31b (127.1.230.129). New cstate: current_term: 2 leader_uuid: "968c62486e7148c4978383fb424be31b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: UNKNOWN } } }
12:12:37.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.698378  2448 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Refusing update from remote peer 968c62486e7148c4978383fb424be31b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
12:12:37.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:37.698773  2690 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Connected to new peer: Peer: permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
12:12:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:37.700654  2249 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b -> Peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Couldn't send request to peer 57953137991b435ea05b795d05ae73c9. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:39.058 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:39.059 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:39.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3207), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [3ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [27ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [29ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [32ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [32ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [48ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [50ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [51ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [64ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [67ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [69ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [72ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [72ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [86ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [88ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [90ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [90ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [187ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [188ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [190ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [190ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [584ms] refreshing cache from master, [584ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [587ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [588ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [590ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [590ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [3204ms] refreshing cache from master, [3204ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [3207ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2056618663(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@4d214d27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291) -> (continuation of Deferred@1767804751 after retry RPC@1855205450) -> (continuation of Deferred@1860711059 after retry RPC@556882276) -> (continuation of Deferred@175033175 after retry RPC@557799687), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6ab5496d -> wakeup thread Time-limited test -> (continuation of Deferred@1185057266 after retry RPC@1133184446) -> (continuation of Deferred@571728359 after retry RPC@1210509853) -> (continuation of Deferred@659587714 after retry RPC@1007625291) -> (continuation of Deferred@1767804751 after retry RPC@1855205450) -> (continuation of Deferred@1860711059 after retry RPC@556882276) -> (continuation of Deferred@175033175 after retry RPC@557799687))) after lookup
12:12:39.060 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:39.061 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] Successfully connected to peer
12:12:39.081 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x8d723c2f, L:/127.0.0.1:32832 - R:/127.1.230.129:39309] using TOKEN/null
12:12:39.094 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.131:38541
12:12:39.096 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.096402  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.175100  2732 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.175253  2732 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.175273  2732 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:39.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.176689  2732 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:39.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.176736  2732 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.131
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.178182  2732 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.2732
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.178414  2732 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.178591  2732 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.180941  2737 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.180974  2738 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.181093  2732 server_base.cc:1061] running on GCE node
12:12:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:39.180948  2740 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.181622  2732 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.181901  2732 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:39.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.183050  2732 hybrid_clock.cc:648] HybridClock initialized: now 1783512759183024 us; error 37 us; skew 500 ppm
12:12:39.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.184114  2732 webserver.cc:533] Webserver started at http://127.1.230.131:33199/ using document root <none> and password file <none>
12:12:39.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.184330  2732 fs_manager.cc:362] Metadata directory not provided
12:12:39.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.184428  2732 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.185676  2732 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:12:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.186293  2746 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.186460  2732 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
12:12:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.186535  2732 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57953137991b435ea05b795d05ae73c9"
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.186910  2732 fs_report.cc:389] FS layout report
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/data
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.195307  2732 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.195608  2732 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.195762  2732 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.195998  2732 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.196475  2753 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:12:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.197258  2732 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.197343  2732 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:12:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.197379  2732 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.197932  2732 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.197988  2732 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:12:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.198024  2753 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap starting.
12:12:39.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.205547  2732 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.131:38541
12:12:39.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.205969  2732 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:39.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.206748  2860 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.131:38541 every 8 connection(s)
12:12:39.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.206986  2753 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Log is configured to *not* fsync() on all Append() calls
12:12:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.210935  2753 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.211311  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 2732
12:12:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.211506  2753 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap complete.
12:12:39.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.212498  2753 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent bootstrapping tablet: real 0.015s	user 0.006s	sys 0.005s
12:12:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.213562  2753 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.213979  2753 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Initialized, Role: FOLLOWER
12:12:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.214125  2753 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.214383  2753 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent starting tablet: real 0.002s	user 0.001s	sys 0.002s
12:12:39.216 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:39.216 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.129:39309
12:12:39.217 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.217236  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 2231
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.217850  2861 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.217941  2861 heartbeater.cc:461] Registering TS with master...
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218132  2861 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218281  2862 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218330  2862 heartbeater.cc:461] Registering TS with master...
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218412  2862 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218441  2863 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218472  2863 heartbeater.cc:461] Registering TS with master...
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218554  2863 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218739  2132 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218791  1989 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.218878  2061 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.219221  2132 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.131:38245
12:12:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:39.219954  2861 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:39.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:39.225 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] handling channelInactive
12:12:39.225 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state READY due to: connection closed
12:12:39.226 [DEBUG - Time-limited test] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:39.227 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.227 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.227 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.238 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:39.238 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:39.238 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2018390255(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@1061b39a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4775d48a -> wakeup thread Time-limited test)) after lookup
12:12:39.239 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:39.240 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.240 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.241 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.258 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:39.258 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:39.259 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [14ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [15ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [16ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [33ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2018390255(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@1061b39a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4775d48a -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868))) after lookup
12:12:39.260 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:39.261 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.261 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.261 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.277 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:39.277 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:39.278 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [14ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [15ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [16ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [33ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [34ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [36ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [36ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [52ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2018390255(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@1061b39a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4775d48a -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327))) after lookup
12:12:39.279 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:39.280 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.280 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.280 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:39.338 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:39.338 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=112), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [14ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [15ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [16ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [33ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [34ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [36ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [36ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [52ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [53ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [55ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [55ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [110ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [112ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2018390255(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@1061b39a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327) -> (continuation of Deferred@364450828 after retry RPC@244970370), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4775d48a -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327) -> (continuation of Deferred@364450828 after retry RPC@244970370))) after lookup
12:12:39.339 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:39.340 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.340 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.340 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.518 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:39.518 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:39.519 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=293), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [14ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [15ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [16ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [33ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [34ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [36ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [36ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [52ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [53ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [55ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [55ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [110ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [112ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [113ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [115ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [115ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [290ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [293ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2018390255(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@1061b39a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327) -> (continuation of Deferred@364450828 after retry RPC@244970370) -> (continuation of Deferred@135240109 after retry RPC@737540677), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4775d48a -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327) -> (continuation of Deferred@364450828 after retry RPC@244970370) -> (continuation of Deferred@135240109 after retry RPC@737540677))) after lookup
12:12:39.520 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:39.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:39.521 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.593968  2874 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 968c62486e7148c4978383fb424be31b)
12:12:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.594126  2874 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.594328  2874 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:40.595391  2383 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.597929  2815 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "57953137991b435ea05b795d05ae73c9" is_pre_election: true
12:12:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.598062  2815 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 010544b667d64cc6b3fa372f3d439d95 in term 1.
12:12:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.598304  2384 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9; no voters: 968c62486e7148c4978383fb424be31b
12:12:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.598412  2874 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Leader pre-election won for term 3
12:12:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.598469  2874 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 968c62486e7148c4978383fb424be31b)
12:12:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.598496  2874 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 2 FOLLOWER]: Advancing to term 3
12:12:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.599198  2874 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.599359  2874 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 3 election: Requested vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.599568  2815 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "57953137991b435ea05b795d05ae73c9"
12:12:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.599659  2815 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 1 FOLLOWER]: Advancing to term 3
12:12:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:40.599972  2383 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.600651  2815 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 010544b667d64cc6b3fa372f3d439d95 in term 3.
12:12:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.600852  2384 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9; no voters: 968c62486e7148c4978383fb424be31b
12:12:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.600931  2874 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Leader election won for term 3
12:12:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.601044  2874 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 LEADER]: Becoming Leader. State: Replica: 010544b667d64cc6b3fa372f3d439d95, State: Running, Role: LEADER
12:12:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.601133  2874 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:40.601936  2132 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 reported cstate change: term changed from 2 to 3, leader changed from 968c62486e7148c4978383fb424be31b (127.1.230.129) to 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130). New cstate: current_term: 3 leader_uuid: "010544b667d64cc6b3fa372f3d439d95" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: UNKNOWN } } }
12:12:41.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.167309  2383 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 -> Peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Couldn't send request to peer 968c62486e7148c4978383fb424be31b. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:41.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.179236  2815 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 3 FOLLOWER]: Refusing update from remote peer 010544b667d64cc6b3fa372f3d439d95: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
12:12:41.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.179584  2892 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
12:12:41.537 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:41.538 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:41.538 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2313), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [14ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [15ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [16ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [33ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [34ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [36ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [36ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [52ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [53ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [55ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [55ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [110ms] refreshing cache from master, [110ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [112ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [113ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [115ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [115ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [290ms] refreshing cache from master, [291ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [293ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [294ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [296ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [296ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2310ms] refreshing cache from master, [2310ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [2312ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2018390255(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@1061b39a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327) -> (continuation of Deferred@364450828 after retry RPC@244970370) -> (continuation of Deferred@135240109 after retry RPC@737540677) -> (continuation of Deferred@959577376 after retry RPC@1024597905), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4775d48a -> wakeup thread Time-limited test -> (continuation of Deferred@870039561 after retry RPC@1524863868) -> (continuation of Deferred@984808126 after retry RPC@2081220327) -> (continuation of Deferred@364450828 after retry RPC@244970370) -> (continuation of Deferred@135240109 after retry RPC@737540677) -> (continuation of Deferred@959577376 after retry RPC@1024597905))) after lookup
12:12:41.539 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:41.540 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] Successfully connected to peer
12:12:41.555 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8be7f0ef, L:/127.0.0.1:36328 - R:/127.1.230.130:42273] using TOKEN/null
12:12:41.567 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.129:39309
12:12:41.568 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:41.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.568624  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:41.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:41.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:41.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:12:41.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:41.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:39309
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34273
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.645049  2906 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.645232  2906 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.645253  2906 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.646881  2906 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.646943  2906 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.129
12:12:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.648460  2906 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:12:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:39309
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34273
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.2906
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.648721  2906 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.648932  2906 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.651432  2911 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.651422  2912 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:41.651444  2914 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.652151  2906 server_base.cc:1061] running on GCE node
12:12:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.652323  2906 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.652545  2906 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.653741  2906 hybrid_clock.cc:648] HybridClock initialized: now 1783512761653708 us; error 46 us; skew 500 ppm
12:12:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.655171  2906 webserver.cc:533] Webserver started at http://127.1.230.129:34273/ using document root <none> and password file <none>
12:12:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.655426  2906 fs_manager.cc:362] Metadata directory not provided
12:12:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.655514  2906 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:41.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.656787  2906 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:12:41.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.657486  2920 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:41.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.657706  2906 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.657766  2906 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "968c62486e7148c4978383fb424be31b"
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.658004  2906 fs_report.cc:389] FS layout report
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/data
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.677165  2906 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.677455  2906 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.677542  2906 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.677801  2906 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.678305  2927 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:12:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.679363  2906 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.679419  2906 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:12:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.679446  2906 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.679999  2906 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.680033  2906 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:12:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.680136  2927 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap starting.
12:12:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.687090  2906 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.129:39309
12:12:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.687201  3034 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.129:39309 every 8 connection(s)
12:12:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.687500  2906 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:41.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.688936  2927 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Log is configured to *not* fsync() on all Append() calls
12:12:41.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.691769  2927 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:41.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.692095  2927 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap complete.
12:12:41.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.693034  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 2906
12:12:41.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.693560  3035 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.693651  3035 heartbeater.cc:461] Registering TS with master...
12:12:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.693857  3035 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.693919  3037 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.693957  3037 heartbeater.cc:461] Registering TS with master...
12:12:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.694032  3037 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.694357  2132 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.694401  2060 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.695247  2132 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.129:43861
12:12:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.696310  3036 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.696377  3036 heartbeater.cc:461] Registering TS with master...
12:12:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.696437  3036 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.696808  1988 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:41.699 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.699707  2927 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent bootstrapping tablet: real 0.020s	user 0.006s	sys 0.007s
12:12:41.700 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.130:42273
12:12:41.700 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.700577  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 2366
12:12:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.702190  2927 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.702446  2927 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 968c62486e7148c4978383fb424be31b, State: Initialized, Role: FOLLOWER
12:12:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.702553  2927 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.702828  2927 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent starting tablet: real 0.003s	user 0.001s	sys 0.001s
12:12:41.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:41.702858  3035 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:41.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:41.706 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] handling channelInactive
12:12:41.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state READY due to: connection closed
12:12:41.707 [DEBUG - Time-limited test] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:41.708 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.708 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.708 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.717 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:41.717 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:41.717 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test)) after lookup
12:12:41.718 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:41.719 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.719 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.719 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.738 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:41.738 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:41.738 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [11ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [12ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [12ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [30ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344))) after lookup
12:12:41.739 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:41.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.740 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.740 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.778 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:41.778 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:41.779 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [11ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [12ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [12ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [30ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [32ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [33ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [70ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546))) after lookup
12:12:41.779 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:41.781 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.781 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.781 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:41.838 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:41.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=131), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [11ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [12ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [12ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [30ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [32ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [33ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [70ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [72ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [74ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [74ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [130ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986))) after lookup
12:12:41.839 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:41.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.840 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.840 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.977 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:41.978 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:41.978 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=271), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [11ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [12ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [12ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [30ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [32ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [33ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [70ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [72ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [74ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [74ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [130ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [132ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [133ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [133ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [268ms] refreshing cache from master, [268ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [270ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016))) after lookup
12:12:41.979 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:41.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.980 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:41.980 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:42.037 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:42.037 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:42.038 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=331), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [11ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [12ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [12ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [30ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [32ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [33ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [70ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [72ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [74ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [74ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [130ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [132ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [133ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [133ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [268ms] refreshing cache from master, [268ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [270ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [271ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [273ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [273ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [328ms] refreshing cache from master, [328ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [330ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016) -> (continuation of Deferred@1963306024 after retry RPC@752877231), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016) -> (continuation of Deferred@1963306024 after retry RPC@752877231))) after lookup
12:12:42.038 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:42.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:42.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:42.040 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:42.698 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:42.698 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:42.698 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=991), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [11ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [12ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [12ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [30ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [32ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [33ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [70ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [72ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [74ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [74ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [130ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [132ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [133ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [133ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [268ms] refreshing cache from master, [268ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [270ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [271ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [273ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [273ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [328ms] refreshing cache from master, [328ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [330ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [331ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [333ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [333ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [988ms] refreshing cache from master, [988ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [990ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016) -> (continuation of Deferred@1963306024 after retry RPC@752877231) -> (continuation of Deferred@1789909200 after retry RPC@970353068), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016) -> (continuation of Deferred@1963306024 after retry RPC@752877231) -> (continuation of Deferred@1789909200 after retry RPC@970353068))) after lookup
12:12:42.699 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:42.700 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:42.700 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:42.700 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:42.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.876870  3048 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:42.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.876987  3048 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:42.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.877245  3048 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:42.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.877729  2921 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:42.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:42.878199  2921 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:42.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.880553  2815 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "57953137991b435ea05b795d05ae73c9" is_pre_election: true
12:12:42.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.880857  2924 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 968c62486e7148c4978383fb424be31b; no voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9
12:12:42.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:42.880982  3048 raft_consensus.cc:2749] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
12:12:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.067391  3053 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 010544b667d64cc6b3fa372f3d439d95)
12:12:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.067535  3053 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.067901  3053 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:43.068361  2747 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:43.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:43.068873  2747 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:43.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.071130  2989 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b" is_pre_election: true
12:12:43.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.071261  2989 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 2.
12:12:43.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.071513  2749 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b; no voters: 010544b667d64cc6b3fa372f3d439d95
12:12:43.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.071691  3053 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 3 FOLLOWER]: Leader pre-election won for term 4
12:12:43.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.071755  3053 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 010544b667d64cc6b3fa372f3d439d95)
12:12:43.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.071772  3053 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 3 FOLLOWER]: Advancing to term 4
12:12:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.072377  3053 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.072531  3053 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 4 election: Requested vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.072690  2989 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b"
12:12:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.072849  2989 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 2 FOLLOWER]: Advancing to term 4
12:12:43.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:43.073124  2747 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.073948  2989 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 4.
12:12:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.074129  2749 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b; no voters: 010544b667d64cc6b3fa372f3d439d95
12:12:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.074220  3053 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 4 FOLLOWER]: Leader election won for term 4
12:12:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.074373  3053 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 4 LEADER]: Becoming Leader. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Running, Role: LEADER
12:12:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.074438  3053 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.075151  2132 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 reported cstate change: term changed from 3 to 4, leader changed from 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130) to 57953137991b435ea05b795d05ae73c9 (127.1.230.131). New cstate: current_term: 4 leader_uuid: "57953137991b435ea05b795d05ae73c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: HEALTHY } } }
12:12:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:43.634987  2747 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 -> Peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Couldn't send request to peer 010544b667d64cc6b3fa372f3d439d95. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:43.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.672035  2989 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 4 FOLLOWER]: Refusing update from remote peer 57953137991b435ea05b795d05ae73c9: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
12:12:43.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:43.672433  3071 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
12:12:44.417 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:44.418 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:44.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=16, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2711), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [11ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [12ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [12ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [30ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [32ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [33ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [33ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [70ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [72ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [74ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [74ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [130ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [132ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [133ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [133ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [268ms] refreshing cache from master, [268ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [270ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [271ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [273ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [273ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [328ms] refreshing cache from master, [328ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [330ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [331ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [333ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [333ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [988ms] refreshing cache from master, [988ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [990ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [992ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [993ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [993ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [2708ms] refreshing cache from master, [2708ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [2710ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1473516705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@500378c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016) -> (continuation of Deferred@1963306024 after retry RPC@752877231) -> (continuation of Deferred@1789909200 after retry RPC@970353068) -> (continuation of Deferred@1344327551 after retry RPC@1521567710), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10eec12b -> wakeup thread Time-limited test -> (continuation of Deferred@1082717451 after retry RPC@646546344) -> (continuation of Deferred@1697835439 after retry RPC@256680546) -> (continuation of Deferred@217282524 after retry RPC@1724291986) -> (continuation of Deferred@1336968983 after retry RPC@502833016) -> (continuation of Deferred@1963306024 after retry RPC@752877231) -> (continuation of Deferred@1789909200 after retry RPC@970353068) -> (continuation of Deferred@1344327551 after retry RPC@1521567710))) after lookup
12:12:44.419 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:44.420 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] Successfully connected to peer
12:12:44.439 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xc5a0cba4, L:/127.0.0.1:46248 - R:/127.1.230.131:38541] using TOKEN/null
12:12:44.453 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.130:42273
12:12:44.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.453567  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:12:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:42273
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46119
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.531383  3090 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.531528  3090 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.531546  3090 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.532936  3090 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.532980  3090 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.130
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.534403  3090 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:42273
12:12:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46119
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3090
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.534574  3090 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.534817  3090 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.537446  3096 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.537420  3098 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:44.537425  3095 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.537992  3090 server_base.cc:1061] running on GCE node
12:12:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.538200  3090 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.538439  3090 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:44.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.539609  3090 hybrid_clock.cc:648] HybridClock initialized: now 1783512764539596 us; error 25 us; skew 500 ppm
12:12:44.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.540683  3090 webserver.cc:533] Webserver started at http://127.1.230.130:46119/ using document root <none> and password file <none>
12:12:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.540896  3090 fs_manager.cc:362] Metadata directory not provided
12:12:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.540993  3090 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.542210  3090 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.542924  3104 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.543133  3090 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.000s
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.543223  3090 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.543514  3090 fs_report.cc:389] FS layout report
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/data
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:44.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.555778  3090 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:44.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.556111  3090 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:44.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.556260  3090 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:44.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.556506  3090 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:44.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.556986  3111 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:12:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.557983  3090 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.558043  3090 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:12:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.558076  3090 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.558632  3090 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.558720  3090 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.001s
12:12:44.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.558874  3111 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap starting.
12:12:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.565460  3090 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.130:42273
12:12:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.565522  3218 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.130:42273 every 8 connection(s)
12:12:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.565806  3090 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.568554  3111 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Log is configured to *not* fsync() on all Append() calls
12:12:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.569818  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 3090
12:12:44.570 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:44.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.571921  3111 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:44.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.572263  3111 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap complete.
12:12:44.575 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:44.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.575461  3219 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:44.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.575564  3219 heartbeater.cc:461] Registering TS with master...
12:12:44.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.575681  3111 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent bootstrapping tablet: real 0.017s	user 0.008s	sys 0.006s
12:12:44.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.575754  3219 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.576330  2132 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.576809  2132 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.130:49987
12:12:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.576835  3111 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.577086  3111 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010544b667d64cc6b3fa372f3d439d95, State: Initialized, Role: FOLLOWER
12:12:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.577188  3111 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.577426  3111 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
12:12:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.577864  3219 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.578884  3220 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.578941  3220 heartbeater.cc:461] Registering TS with master...
12:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.578985  3220 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.579252  1989 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:44.582 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.131:38541
12:12:44.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:44.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.583225  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 2732
12:12:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.589929  3221 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.590163  3221 heartbeater.cc:461] Registering TS with master...
12:12:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.590345  3221 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:44.590817  2061 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:44.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:44.592 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] handling channelInactive
12:12:44.592 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state READY due to: connection closed
12:12:44.596 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:44.597 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.597 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.597 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.618 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:44.618 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:44.618 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [5ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [5ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2088627736(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6a5d0fb7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66122f7b -> wakeup thread Time-limited test)) after lookup
12:12:44.619 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:44.620 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.620 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.620 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.637 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:44.637 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:44.638 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [5ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [5ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [27ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [28ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [28ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2088627736(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6a5d0fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66122f7b -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335))) after lookup
12:12:44.639 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:44.640 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.640 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.640 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.657 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:44.657 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:44.658 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [5ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [5ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [27ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [28ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [28ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [46ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [48ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [48ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2088627736(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6a5d0fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66122f7b -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658))) after lookup
12:12:44.658 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:44.659 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.659 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.659 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:44.798 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:44.798 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=206), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [5ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [5ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [27ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [28ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [28ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [46ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [48ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [48ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [66ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [67ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [68ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [203ms] refreshing cache from master, [203ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [205ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2088627736(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6a5d0fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658) -> (continuation of Deferred@2025053616 after retry RPC@952045802), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66122f7b -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658) -> (continuation of Deferred@2025053616 after retry RPC@952045802))) after lookup
12:12:44.799 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:44.800 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.800 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:44.800 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:45.518 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:45.518 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:45.519 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=927), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [5ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [5ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [27ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [28ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [28ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [46ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [48ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [48ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [66ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [67ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [68ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [203ms] refreshing cache from master, [203ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [205ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [206ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [208ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [208ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [923ms] refreshing cache from master, [924ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [926ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2088627736(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6a5d0fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658) -> (continuation of Deferred@2025053616 after retry RPC@952045802) -> (continuation of Deferred@1951659906 after retry RPC@883659170), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66122f7b -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658) -> (continuation of Deferred@2025053616 after retry RPC@952045802) -> (continuation of Deferred@1951659906 after retry RPC@883659170))) after lookup
12:12:45.519 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:45.520 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:45.520 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:45.520 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.768121  3233 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.768281  3233 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.768635  3233 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:45.769066  3108 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:45.769554  3108 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:45.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.771632  2989 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b" is_pre_election: true
12:12:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.771894  3107 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95; no voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b
12:12:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.772020  3233 raft_consensus.cc:2749] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
12:12:45.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.952684  3238 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:12:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.952788  3238 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.952976  3238 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:45.953878  2924 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.956179  3173 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95" is_pre_election: true
12:12:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.956288  3173 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 3.
12:12:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.956521  2921 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 57953137991b435ea05b795d05ae73c9
12:12:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.956647  3238 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 4 FOLLOWER]: Leader pre-election won for term 5
12:12:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.956691  3238 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 4 FOLLOWER]: Starting leader election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:12:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.956705  3238 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 4 FOLLOWER]: Advancing to term 5
12:12:45.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.957254  3238 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:45.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.957398  3238 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 5 election: Requested vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:45.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.957567  3173 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:45.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.957667  3173 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 3 FOLLOWER]: Advancing to term 5
12:12:45.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:45.958026  2924 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:45.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.958844  3173 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 5.
12:12:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.959019  2921 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 57953137991b435ea05b795d05ae73c9
12:12:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.959146  3238 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 5 FOLLOWER]: Leader election won for term 5
12:12:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.959288  3238 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 5 LEADER]: Becoming Leader. State: Replica: 968c62486e7148c4978383fb424be31b, State: Running, Role: LEADER
12:12:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.959360  3238 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:45.960059  2132 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b reported cstate change: term changed from 4 to 5, leader changed from 57953137991b435ea05b795d05ae73c9 (127.1.230.131) to 968c62486e7148c4978383fb424be31b (127.1.230.129). New cstate: current_term: 5 leader_uuid: "968c62486e7148c4978383fb424be31b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: UNKNOWN } } }
12:12:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:46.396901  3173 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 5 FOLLOWER]: Refusing update from remote peer 968c62486e7148c4978383fb424be31b: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
12:12:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:46.397253  3238 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Connected to new peer: Peer: permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
12:12:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:46.400391  2924 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b -> Peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Couldn't send request to peer 57953137991b435ea05b795d05ae73c9. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:47.997 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:47.997 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:47.998 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=3405), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [5ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [5ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [27ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [28ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [28ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [45ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [46ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [48ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [48ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [66ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [67ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [68ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [203ms] refreshing cache from master, [203ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [205ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [206ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [208ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [208ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [923ms] refreshing cache from master, [924ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [926ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [927ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [928ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [928ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [3403ms] refreshing cache from master, [3403ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [3405ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@2088627736(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6a5d0fb7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658) -> (continuation of Deferred@2025053616 after retry RPC@952045802) -> (continuation of Deferred@1951659906 after retry RPC@883659170) -> (continuation of Deferred@67713206 after retry RPC@1816633108), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66122f7b -> wakeup thread Time-limited test -> (continuation of Deferred@45725004 after retry RPC@1965501335) -> (continuation of Deferred@1477279175 after retry RPC@347774658) -> (continuation of Deferred@2025053616 after retry RPC@952045802) -> (continuation of Deferred@1951659906 after retry RPC@883659170) -> (continuation of Deferred@67713206 after retry RPC@1816633108))) after lookup
12:12:47.998 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:47.999 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] Successfully connected to peer
12:12:48.017 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x980842fc, L:/127.0.0.1:38906 - R:/127.1.230.129:39309] using TOKEN/null
12:12:48.027 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.131:38541
12:12:48.028 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.028724  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.106238  3273 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.106458  3273 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.106490  3273 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.108084  3273 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.108170  3273 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.131
12:12:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.109709  3273 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3273
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.109959  3273 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.110204  3273 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.112847  3278 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.112963  3279 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:48.112833  3281 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.113126  3273 server_base.cc:1061] running on GCE node
12:12:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.113303  3273 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.113503  3273 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.114655  3273 hybrid_clock.cc:648] HybridClock initialized: now 1783512768114624 us; error 44 us; skew 500 ppm
12:12:48.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.115761  3273 webserver.cc:533] Webserver started at http://127.1.230.131:33199/ using document root <none> and password file <none>
12:12:48.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.115924  3273 fs_manager.cc:362] Metadata directory not provided
12:12:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.115968  3273 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:48.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.117015  3273 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.001s
12:12:48.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.117547  3287 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:48.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.117687  3273 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.117727  3273 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57953137991b435ea05b795d05ae73c9"
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.117971  3273 fs_report.cc:389] FS layout report
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/data
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.128858  3273 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.129105  3273 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.129194  3273 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.129366  3273 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.129765  3294 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:12:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.130622  3273 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.130677  3273 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:12:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.130728  3273 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:48.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.131206  3273 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:48.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.131235  3273 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:12:48.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.131342  3294 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap starting.
12:12:48.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.137743  3273 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.131:38541
12:12:48.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.137787  3401 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.131:38541 every 8 connection(s)
12:12:48.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.138103  3273 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:48.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.141510  3294 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Log is configured to *not* fsync() on all Append() calls
12:12:48.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.143179  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 3273
12:12:48.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.144419  3404 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.144515  3404 heartbeater.cc:461] Registering TS with master...
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.144706  3404 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.145117  3294 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.145406  3294 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap complete.
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.145963  2061 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.146490  3294 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent bootstrapping tablet: real 0.015s	user 0.001s	sys 0.010s
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.146768  3403 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.146832  3403 heartbeater.cc:461] Registering TS with master...
12:12:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.146932  3403 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:48.148 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:48.149 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.129:39309
12:12:48.149 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.147300  1989 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.147430  3402 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.147469  3402 heartbeater.cc:461] Registering TS with master...
12:12:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.147552  3402 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.147786  2132 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.147810  3294 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.148052  3294 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Initialized, Role: FOLLOWER
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.148149  3294 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.148278  2132 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.131:32887
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.148386  3294 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent starting tablet: real 0.002s	user 0.005s	sys 0.000s
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.148941  3402 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:48.149570  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 2906
12:12:48.155 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] handling channelInactive
12:12:48.155 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state READY due to: connection closed
12:12:48.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:48.156 [DEBUG - Time-limited test] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:48.157 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.157 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.157 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.177 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:48.177 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:48.178 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [1ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@734414183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@11aa18ca -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6cb9addf -> wakeup thread Time-limited test)) after lookup
12:12:48.178 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:48.179 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.179 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.180 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.197 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:48.197 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:48.198 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [1ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [24ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [24ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@734414183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@11aa18ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6cb9addf -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005))) after lookup
12:12:48.198 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:48.199 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.199 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.200 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.217 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:48.218 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:48.222 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [1ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [24ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [24ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [61ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@734414183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@11aa18ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6cb9addf -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145))) after lookup
12:12:48.223 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:48.224 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.224 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.225 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.278 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:48.278 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:48.278 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=122), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [1ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [24ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [24ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [61ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [67ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [68ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [69ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [121ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@734414183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@11aa18ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145) -> (continuation of Deferred@667058328 after retry RPC@1781417771), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6cb9addf -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145) -> (continuation of Deferred@667058328 after retry RPC@1781417771))) after lookup
12:12:48.279 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:48.280 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.280 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.281 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.517 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:48.518 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:48.518 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=361), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [1ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [24ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [24ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [61ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [67ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [68ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [69ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [121ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [123ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [125ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [125ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [359ms] refreshing cache from master, [359ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [361ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@734414183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@11aa18ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145) -> (continuation of Deferred@667058328 after retry RPC@1781417771) -> (continuation of Deferred@107565171 after retry RPC@510023818), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6cb9addf -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145) -> (continuation of Deferred@667058328 after retry RPC@1781417771) -> (continuation of Deferred@107565171 after retry RPC@510023818))) after lookup
12:12:48.518 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:48.519 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.520 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:48.520 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.527971  3415 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 968c62486e7148c4978383fb424be31b)
12:12:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.528066  3415 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.528235  3415 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.529031  3107 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.531435  3356 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "57953137991b435ea05b795d05ae73c9" is_pre_election: true
12:12:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.531561  3356 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 010544b667d64cc6b3fa372f3d439d95 in term 4.
12:12:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.531872  3108 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9; no voters: 968c62486e7148c4978383fb424be31b
12:12:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.532015  3415 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 5 FOLLOWER]: Leader pre-election won for term 6
12:12:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.532073  3415 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 968c62486e7148c4978383fb424be31b)
12:12:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.532101  3415 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 5 FOLLOWER]: Advancing to term 6
12:12:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.532755  3415 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:49.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.532922  3415 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 6 election: Requested vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:49.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.533134  3356 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "57953137991b435ea05b795d05ae73c9"
12:12:49.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.533217  3356 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 4 FOLLOWER]: Advancing to term 6
12:12:49.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.533473  3107 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.534224  3356 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 010544b667d64cc6b3fa372f3d439d95 in term 6.
12:12:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.534425  3108 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9; no voters: 968c62486e7148c4978383fb424be31b
12:12:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.534525  3415 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 6 FOLLOWER]: Leader election won for term 6
12:12:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.534668  3415 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 6 LEADER]: Becoming Leader. State: Replica: 010544b667d64cc6b3fa372f3d439d95, State: Running, Role: LEADER
12:12:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.534781  3415 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.535487  2132 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 reported cstate change: term changed from 5 to 6, leader changed from 968c62486e7148c4978383fb424be31b (127.1.230.129) to 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130). New cstate: current_term: 6 leader_uuid: "010544b667d64cc6b3fa372f3d439d95" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: UNKNOWN } } }
12:12:49.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:49.797 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:49.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1641), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [1ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [24ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [24ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [61ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [67ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [68ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [69ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [121ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [123ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [125ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [125ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [359ms] refreshing cache from master, [359ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [361ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [362ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [364ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [364ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1639ms] refreshing cache from master, [1639ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [1640ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@734414183(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@11aa18ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145) -> (continuation of Deferred@667058328 after retry RPC@1781417771) -> (continuation of Deferred@107565171 after retry RPC@510023818) -> (continuation of Deferred@1398736108 after retry RPC@2144445445), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6cb9addf -> wakeup thread Time-limited test -> (continuation of Deferred@1709484411 after retry RPC@1150429005) -> (continuation of Deferred@956947098 after retry RPC@1827945145) -> (continuation of Deferred@667058328 after retry RPC@1781417771) -> (continuation of Deferred@107565171 after retry RPC@510023818) -> (continuation of Deferred@1398736108 after retry RPC@2144445445))) after lookup
12:12:49.798 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:49.799 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] Successfully connected to peer
12:12:49.816 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x94a88ec4, L:/127.0.0.1:52440 - R:/127.1.230.130:42273] using TOKEN/null
12:12:49.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.817971  3356 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 6 FOLLOWER]: Refusing update from remote peer 010544b667d64cc6b3fa372f3d439d95: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 22. (index mismatch)
12:12:49.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.818053  3107 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 -> Peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Couldn't send request to peer 968c62486e7148c4978383fb424be31b. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:49.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.818262  3420 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
12:12:49.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.820379  3433 mvcc.cc:204] Tried to move back new op lower bound from 7305268305171779584 to 7305268304014852096. Current Snapshot: MvccSnapshot[applied={T|T < 7305268305171779584}]
12:12:49.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.821592  3435 mvcc.cc:204] Tried to move back new op lower bound from 7305268305171779584 to 7305268304014852096. Current Snapshot: MvccSnapshot[applied={T|T < 7305268305171779584}]
12:12:49.830 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.129:39309
12:12:49.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.832188  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:39309
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34273
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:49.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.919898  3441 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:49.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.920064  3441 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:49.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.920083  3441 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:49.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.921618  3441 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:49.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.921713  3441 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.129
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.923489  3441 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:39309
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:49.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34273
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3441
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.923673  3441 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:49.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.923859  3441 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.926249  3446 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.926271  3447 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.926483  3441 server_base.cc:1061] running on GCE node
12:12:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:49.926265  3449 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.926859  3441 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:49.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.927098  3441 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.928258  3441 hybrid_clock.cc:648] HybridClock initialized: now 1783512769928229 us; error 42 us; skew 500 ppm
12:12:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.929309  3441 webserver.cc:533] Webserver started at http://127.1.230.129:34273/ using document root <none> and password file <none>
12:12:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.929545  3441 fs_manager.cc:362] Metadata directory not provided
12:12:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.929633  3441 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.930876  3441 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:12:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.931699  3455 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.931903  3441 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.931990  3441 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "968c62486e7148c4978383fb424be31b"
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.932374  3441 fs_report.cc:389] FS layout report
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/data
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.983888  3441 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.984169  3441 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.984256  3441 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.984441  3441 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:49.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.984865  3462 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:12:49.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.985698  3441 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:49.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.985746  3441 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:12:49.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.985775  3441 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.986287  3441 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.986317  3441 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:12:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.986418  3462 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap starting.
12:12:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.992797  3441 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.129:39309
12:12:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.992846  3569 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.129:39309 every 8 connection(s)
12:12:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.993149  3441 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:12:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:49.996868  3462 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Log is configured to *not* fsync() on all Append() calls
12:12:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.000191  3462 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.000504  3462 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap complete.
12:12:50.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.001402  3462 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent bootstrapping tablet: real 0.015s	user 0.012s	sys 0.000s
12:12:50.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.001922  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 3441
12:12:50.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:50.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.002446  3462 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:50.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.002728  3462 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 968c62486e7148c4978383fb424be31b, State: Initialized, Role: FOLLOWER
12:12:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.002831  3462 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.003083  3462 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent starting tablet: real 0.002s	user 0.004s	sys 0.000s
12:12:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.003777  3572 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.003865  3572 heartbeater.cc:461] Registering TS with master...
12:12:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004056  3572 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004146  3571 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004208  3571 heartbeater.cc:461] Registering TS with master...
12:12:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004295  3571 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004153  3570 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004657  1989 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004956  2061 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.004534  3570 heartbeater.cc:461] Registering TS with master...
12:12:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.005489  3570 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.005725  2133 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:12:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.006112  2133 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.129:46005
12:12:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.006608  3570 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:50.008 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:50.009 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.130:42273
12:12:50.009 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:50.009603  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 3090
12:12:50.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:50.014 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] handling channelInactive
12:12:50.015 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state READY due to: connection closed
12:12:50.035 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:50.036 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.037 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.037 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.057 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:50.057 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:50.057 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=3, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [23ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@164187452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@534a459c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a009db3 -> wakeup thread Time-limited test)) after lookup
12:12:50.058 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:50.059 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.059 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.059 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.078 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:50.078 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:50.078 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=5, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=63), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [23ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [44ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [45ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [45ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [61ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [63ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@164187452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@534a459c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a009db3 -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484))) after lookup
12:12:50.079 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:50.080 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.080 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.080 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.138 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:50.139 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:50.139 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=7, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=124), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [23ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [44ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [45ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [45ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [61ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [63ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [64ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [66ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [66ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [121ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [124ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@164187452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@534a459c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a009db3 -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020))) after lookup
12:12:50.139 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:50.140 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.140 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.141 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.298 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:50.298 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:50.298 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=9, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=283), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [23ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [44ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [45ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [45ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [61ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [63ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [64ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [66ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [66ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [121ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [124ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [125ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [126ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [127ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [281ms] refreshing cache from master, [281ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [283ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@164187452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@534a459c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020) -> (continuation of Deferred@1368684277 after retry RPC@1003197694), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a009db3 -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020) -> (continuation of Deferred@1368684277 after retry RPC@1003197694))) after lookup
12:12:50.299 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:50.300 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.300 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:50.300 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:51.097 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:51.097 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:51.098 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=11, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1083), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [23ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [44ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [45ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [45ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [61ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [63ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [64ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [66ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [66ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [121ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [124ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [125ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [126ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [127ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [281ms] refreshing cache from master, [281ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [283ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [285ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [286ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [286ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1081ms] refreshing cache from master, [1081ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [1083ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@164187452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@534a459c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020) -> (continuation of Deferred@1368684277 after retry RPC@1003197694) -> (continuation of Deferred@1280152454 after retry RPC@726045333), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a009db3 -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020) -> (continuation of Deferred@1368684277 after retry RPC@1003197694) -> (continuation of Deferred@1280152454 after retry RPC@726045333))) after lookup
12:12:51.099 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:12:51.099 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:51.100 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:51.100 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:12:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.330440  3585 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 010544b667d64cc6b3fa372f3d439d95)
12:12:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.330641  3585 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.331073  3585 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.331481  3288 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.331974  3288 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.334537  3523 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b" is_pre_election: true
12:12:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.334710  3523 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 5.
12:12:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.334924  3290 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b; no voters: 010544b667d64cc6b3fa372f3d439d95
12:12:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.335088  3585 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 6 FOLLOWER]: Leader pre-election won for term 7
12:12:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.335152  3585 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 010544b667d64cc6b3fa372f3d439d95)
12:12:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.335175  3585 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 6 FOLLOWER]: Advancing to term 7
12:12:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.335979  3585 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.336127  3585 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 7 election: Requested vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.336330  3523 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b"
12:12:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.336421  3523 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 5 FOLLOWER]: Advancing to term 7
12:12:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.336591  3288 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:12:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.337550  3523 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 7.
12:12:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.337850  3290 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b; no voters: 010544b667d64cc6b3fa372f3d439d95
12:12:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.337983  3585 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Leader election won for term 7
12:12:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.338140  3585 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 LEADER]: Becoming Leader. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Running, Role: LEADER
12:12:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.338238  3585 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.338999  2133 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 reported cstate change: term changed from 6 to 7, leader changed from 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130) to 57953137991b435ea05b795d05ae73c9 (127.1.230.131). New cstate: current_term: 7 leader_uuid: "57953137991b435ea05b795d05ae73c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: HEALTHY } } }
12:12:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:51.800083  3288 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 -> Peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Couldn't send request to peer 010544b667d64cc6b3fa372f3d439d95. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.909809  3523 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 7 FOLLOWER]: Refusing update from remote peer 57953137991b435ea05b795d05ae73c9: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
12:12:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:51.910219  3585 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
12:12:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:54.203040  3288 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 -> Peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Couldn't send request to peer 010544b667d64cc6b3fa372f3d439d95. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
12:12:55.117 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:55.118 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:55.118 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=13, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=5103), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1ms] delaying RPC due to: Illegal state: connection is terminated, [21ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [23ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [44ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [45ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [45ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [61ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [63ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [64ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [66ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [66ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [121ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [124ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [125ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [126ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [127ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [281ms] refreshing cache from master, [281ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [283ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [285ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [286ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [286ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1081ms] refreshing cache from master, [1081ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [1083ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [1084ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [1086ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1086ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [5101ms] refreshing cache from master, [5101ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [5103ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@164187452(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@534a459c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020) -> (continuation of Deferred@1368684277 after retry RPC@1003197694) -> (continuation of Deferred@1280152454 after retry RPC@726045333) -> (continuation of Deferred@134349517 after retry RPC@399506274), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a009db3 -> wakeup thread Time-limited test -> (continuation of Deferred@2144293465 after retry RPC@923613484) -> (continuation of Deferred@1673555177 after retry RPC@1037762020) -> (continuation of Deferred@1368684277 after retry RPC@1003197694) -> (continuation of Deferred@1280152454 after retry RPC@726045333) -> (continuation of Deferred@134349517 after retry RPC@399506274))) after lookup
12:12:55.119 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:55.120 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] Successfully connected to peer
12:12:55.148 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x4dd54065, L:/127.0.0.1:53632 - R:/127.1.230.131:38541] using TOKEN/null
12:12:55.165 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.130:42273
12:12:55.166 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.166924  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:42273
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46119
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.244987  3624 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.245160  3624 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.245182  3624 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.246598  3624 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.246645  3624 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.130
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.248167  3624 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:42273
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46119
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3624
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.248348  3624 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.248521  3624 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.251500  3629 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.251897  3624 server_base.cc:1061] running on GCE node
12:12:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.251508  3630 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:55.251504  3632 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.252485  3624 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.252722  3624 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.253887  3624 hybrid_clock.cc:648] HybridClock initialized: now 1783512775253869 us; error 33 us; skew 500 ppm
12:12:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.255067  3624 webserver.cc:533] Webserver started at http://127.1.230.130:46119/ using document root <none> and password file <none>
12:12:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.255319  3624 fs_manager.cc:362] Metadata directory not provided
12:12:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.255394  3624 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.256614  3624 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:12:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.257313  3638 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.257488  3624 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
12:12:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.257591  3624 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.257890  3624 fs_report.cc:389] FS layout report
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/data
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:55.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.283726  3624 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.284089  3624 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.284256  3624 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.284509  3624 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.285003  3645 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:12:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.285825  3624 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.285907  3624 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:12:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.285948  3624 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.286516  3624 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.286576  3624 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:12:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.286602  3645 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap starting.
12:12:55.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.293978  3624 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.130:42273
12:12:55.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.294368  3624 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:12:55.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.294451  3752 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.130:42273 every 8 connection(s)
12:12:55.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.297860  3645 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Log is configured to *not* fsync() on all Append() calls
12:12:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.300947  3645 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.301183  3645 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap complete.
12:12:55.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.301978  3645 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent bootstrapping tablet: real 0.015s	user 0.008s	sys 0.004s
12:12:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.302850  3645 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.303040  3645 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010544b667d64cc6b3fa372f3d439d95, State: Initialized, Role: FOLLOWER
12:12:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.303145  3645 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.303198  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 3624
12:12:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.303435  3645 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent starting tablet: real 0.001s	user 0.004s	sys 0.000s
12:12:55.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.304674  3753 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.304720  3754 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.304764  3753 heartbeater.cc:461] Registering TS with master...
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.304791  3754 heartbeater.cc:461] Registering TS with master...
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.304962  3753 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.305044  3755 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.305091  3755 heartbeater.cc:461] Registering TS with master...
12:12:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.305181  3755 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.305946  2060 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.306174  2133 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.304961  3754 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.306704  2133 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.130:35805
12:12:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.307178  1989 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.307405  3753 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:55.308 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:55.310 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.131:38541
12:12:55.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:55.311446  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 3273
12:12:55.317 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:55.317 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] handling channelInactive
12:12:55.317 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state READY due to: connection closed
12:12:55.335 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:55.336 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.337 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.337 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.357 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:55.357 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:55.358 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=3, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [20ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [20ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@71309705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6cbce688 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58687387 -> wakeup thread Time-limited test)) after lookup
12:12:55.358 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:55.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.359 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.377 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:55.378 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:55.378 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=5, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [20ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [20ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [41ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [42ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@71309705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6cbce688 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58687387 -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812))) after lookup
12:12:55.379 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:55.379 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.379 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.380 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.417 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:55.418 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:55.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=7, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=101), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [20ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [20ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [41ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [42ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [61ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [62ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [63ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [100ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@71309705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6cbce688 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58687387 -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536))) after lookup
12:12:55.419 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:55.420 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.420 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.420 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.597 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:55.597 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:55.597 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=9, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=280), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [20ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [20ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [41ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [42ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [61ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [62ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [63ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [100ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [102ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [103ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [103ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [278ms] refreshing cache from master, [278ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [279ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@71309705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6cbce688 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536) -> (continuation of Deferred@1203640785 after retry RPC@944115551), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58687387 -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536) -> (continuation of Deferred@1203640785 after retry RPC@944115551))) after lookup
12:12:55.598 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:55.599 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.599 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.599 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.677 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:55.677 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:12:55.677 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=11, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=360), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [20ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [20ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [41ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [42ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [61ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [62ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [63ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [100ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [102ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [103ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [103ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [278ms] refreshing cache from master, [278ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [279ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [280ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [282ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [282ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [358ms] refreshing cache from master, [358ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [359ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@71309705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6cbce688 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536) -> (continuation of Deferred@1203640785 after retry RPC@944115551) -> (continuation of Deferred@1976611254 after retry RPC@2103731364), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58687387 -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536) -> (continuation of Deferred@1203640785 after retry RPC@944115551) -> (continuation of Deferred@1976611254 after retry RPC@2103731364))) after lookup
12:12:55.678 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:12:55.679 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.679 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:55.679 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:12:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.665951  3766 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:12:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.666055  3766 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.666379  3766 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:56.666874  3459 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:56.667328  3459 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:56.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.670495  3707 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95" is_pre_election: true
12:12:56.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.670683  3707 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 6.
12:12:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.671088  3456 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 57953137991b435ea05b795d05ae73c9
12:12:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.671265  3766 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 7 FOLLOWER]: Leader pre-election won for term 8
12:12:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.671337  3766 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 7 FOLLOWER]: Starting leader election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:12:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.671360  3766 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 7 FOLLOWER]: Advancing to term 8
12:12:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.672080  3766 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.672266  3766 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 8 election: Requested vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.672463  3707 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95"
12:12:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.672586  3707 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 6 FOLLOWER]: Advancing to term 8
12:12:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:56.673028  3459 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:12:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.673882  3707 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 8.
12:12:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.674093  3456 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 57953137991b435ea05b795d05ae73c9
12:12:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.674207  3766 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 8 FOLLOWER]: Leader election won for term 8
12:12:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.674377  3766 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 8 LEADER]: Becoming Leader. State: Replica: 968c62486e7148c4978383fb424be31b, State: Running, Role: LEADER
12:12:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.674489  3766 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:56.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:56.675251  2133 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b reported cstate change: term changed from 7 to 8, leader changed from 57953137991b435ea05b795d05ae73c9 (127.1.230.131) to 968c62486e7148c4978383fb424be31b (127.1.230.129). New cstate: current_term: 8 leader_uuid: "968c62486e7148c4978383fb424be31b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: UNKNOWN } } }
12:12:57.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.074924  3707 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 8 FOLLOWER]: Refusing update from remote peer 968c62486e7148c4978383fb424be31b: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
12:12:57.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:57.075290  3766 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Connected to new peer: Peer: permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
12:12:57.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:57.078438  3459 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b -> Peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Couldn't send request to peer 57953137991b435ea05b795d05ae73c9. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:12:58.437 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:58.438 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:58.438 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=13, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3121), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [20ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [20ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [41ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [42ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [42ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [61ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [62ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [63ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [100ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [102ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [103ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [103ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [278ms] refreshing cache from master, [278ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [279ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [280ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [282ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [282ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [358ms] refreshing cache from master, [358ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [359ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [361ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [362ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [362ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [3118ms] refreshing cache from master, [3118ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [3120ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@71309705(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@6cbce688 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536) -> (continuation of Deferred@1203640785 after retry RPC@944115551) -> (continuation of Deferred@1976611254 after retry RPC@2103731364) -> (continuation of Deferred@1925050887 after retry RPC@748874160), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@58687387 -> wakeup thread Time-limited test -> (continuation of Deferred@687156886 after retry RPC@857375812) -> (continuation of Deferred@597430215 after retry RPC@533469536) -> (continuation of Deferred@1203640785 after retry RPC@944115551) -> (continuation of Deferred@1976611254 after retry RPC@2103731364) -> (continuation of Deferred@1925050887 after retry RPC@748874160))) after lookup
12:12:58.439 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:58.440 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] Successfully connected to peer
12:12:58.467 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x0e0bc136, L:/127.0.0.1:56036 - R:/127.1.230.129:39309] using TOKEN/null
12:12:58.482 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.131:38541
12:12:58.482 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.482831  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:12:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.563786  3804 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:12:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.563961  3804 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:12:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.563979  3804 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:12:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.565414  3804 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:12:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.565460  3804 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.131
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.566944  3804 tablet_server_runner.cc:78] Tablet server non-default flags:
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3804
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.567121  3804 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:12:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.567307  3804 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:12:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.569855  3810 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.569871  3809 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:58.570004  3812 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:12:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.570195  3804 server_base.cc:1061] running on GCE node
12:12:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.570348  3804 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:12:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.570593  3804 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:12:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.571751  3804 hybrid_clock.cc:648] HybridClock initialized: now 1783512778571764 us; error 92 us; skew 500 ppm
12:12:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.572997  3804 webserver.cc:533] Webserver started at http://127.1.230.131:33199/ using document root <none> and password file <none>
12:12:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.573230  3804 fs_manager.cc:362] Metadata directory not provided
12:12:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.573302  3804 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:12:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.574456  3804 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.575160  3818 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.575323  3804 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.575398  3804 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57953137991b435ea05b795d05ae73c9"
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.575692  3804 fs_report.cc:389] FS layout report
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:12:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/data
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:12:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:12:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.590211  3804 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:12:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.590550  3804 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:12:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.590739  3804 kserver.cc:163] Server-wide thread pool size limit: 3276
12:12:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.591041  3804 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:12:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.591579  3825 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:12:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.592401  3804 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:12:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.592464  3804 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:12:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.592499  3804 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:12:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.593088  3804 ts_tablet_manager.cc:616] Registered 1 tablets
12:12:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.593143  3804 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:12:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.593207  3825 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap starting.
12:12:58.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.601130  3804 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.131:38541
12:12:58.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.601554  3804 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:12:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.601708  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 3804
12:12:58.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.601533  3932 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.131:38541 every 8 connection(s)
12:12:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.602650  3825 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Log is configured to *not* fsync() on all Append() calls
12:12:58.607 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:12:58.607 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.129:39309
12:12:58.608 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:12:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.608378  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 3441
12:12:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.606690  3825 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:12:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.609757  3825 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap complete.
12:12:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.610900  3825 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent bootstrapping tablet: real 0.018s	user 0.005s	sys 0.009s
12:12:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.611974  3825 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.612224  3825 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Initialized, Role: FOLLOWER
12:12:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.612315  3825 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.612506  3825 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent starting tablet: real 0.002s	user 0.001s	sys 0.001s
12:12:58.615 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] handling channelInactive
12:12:58.615 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state READY due to: connection closed
12:12:58.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:12:58.617 [DEBUG - Time-limited test] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:58.618 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.618 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.618 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.619936  3933 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:12:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620052  3933 heartbeater.cc:461] Registering TS with master...
12:12:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620275  3934 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:12:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620373  3934 heartbeater.cc:461] Registering TS with master...
12:12:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620257  3933 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:12:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620467  3934 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:12:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620935  2133 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.620977  1989 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.621333  3935 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:12:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.621390  3935 heartbeater.cc:461] Registering TS with master...
12:12:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.621439  2133 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.131:52319
12:12:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.621477  3935 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:12:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.622138  2060 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:58.622287  3933 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:12:58.637 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:58.637 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:58.638 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@406447331(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@236d613d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f1cf264 -> wakeup thread Time-limited test)) after lookup
12:12:58.638 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:58.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.639 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.657 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:58.657 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:58.658 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [23ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [23ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@406447331(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@236d613d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f1cf264 -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307))) after lookup
12:12:58.659 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:58.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.660 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.697 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:58.698 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:58.698 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [23ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [23ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@406447331(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@236d613d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f1cf264 -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829))) after lookup
12:12:58.699 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:58.700 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.700 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.700 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.798 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:58.798 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:58.799 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=182), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [23ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [23ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [82ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [84ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [84ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [179ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [182ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@406447331(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@236d613d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f1cf264 -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917))) after lookup
12:12:58.799 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:58.800 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.801 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:58.998 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:58.998 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:58.999 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=382), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [23ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [23ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [82ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [84ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [84ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [179ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [182ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [183ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [185ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [185ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [379ms] refreshing cache from master, [380ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [382ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@406447331(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@236d613d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917) -> (continuation of Deferred@794873505 after retry RPC@1756923450), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f1cf264 -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917) -> (continuation of Deferred@794873505 after retry RPC@1756923450))) after lookup
12:12:58.999 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:59.000 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:59.000 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:59.001 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:59.518 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:12:59.518 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:12:59.518 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=902), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [23ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [23ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [82ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [84ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [84ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [179ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [182ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [183ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [185ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [185ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [379ms] refreshing cache from master, [380ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [382ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [383ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [384ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [385ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [899ms] refreshing cache from master, [900ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [901ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@406447331(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@236d613d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917) -> (continuation of Deferred@794873505 after retry RPC@1756923450) -> (continuation of Deferred@1765323101 after retry RPC@1167946518), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f1cf264 -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917) -> (continuation of Deferred@794873505 after retry RPC@1756923450) -> (continuation of Deferred@1765323101 after retry RPC@1167946518))) after lookup
12:12:59.519 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:12:59.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:59.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state CONNECTING due to: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:59.521 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 968c62486e7148c4978383fb424be31b(127.1.230.129:39309) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309
12:12:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.841068  3946 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:12:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.841188  3946 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.841498  3946 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:12:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.841897  3821 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.842507  3821 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:59.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.845204  3707 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95" is_pre_election: true
12:12:59.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.845534  3819 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 57953137991b435ea05b795d05ae73c9; no voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b
12:12:59.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.845687  3946 raft_consensus.cc:2749] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
12:12:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.982012  3951 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 968c62486e7148c4978383fb424be31b)
12:12:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.982107  3951 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.982406  3951 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.982954  3641 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.983445  3641 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.985563  3887 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "57953137991b435ea05b795d05ae73c9" is_pre_election: true
12:12:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.985697  3887 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 010544b667d64cc6b3fa372f3d439d95 in term 7.
12:12:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.985947  3642 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9; no voters: 968c62486e7148c4978383fb424be31b
12:12:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.986078  3951 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 8 FOLLOWER]: Leader pre-election won for term 9
12:12:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.986155  3951 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 968c62486e7148c4978383fb424be31b)
12:12:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.986183  3951 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 8 FOLLOWER]: Advancing to term 9
12:12:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.986860  3951 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.987015  3951 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 9 election: Requested vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:12:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.987183  3887 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "57953137991b435ea05b795d05ae73c9"
12:12:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.987293  3887 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 7 FOLLOWER]: Advancing to term 9
12:12:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:12:59.987587  3641 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111)
12:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.988327  3887 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 010544b667d64cc6b3fa372f3d439d95 in term 9.
12:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.988505  3642 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 57953137991b435ea05b795d05ae73c9; no voters: 968c62486e7148c4978383fb424be31b
12:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.988577  3951 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Leader election won for term 9
12:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.988742  3951 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 LEADER]: Becoming Leader. State: Replica: 010544b667d64cc6b3fa372f3d439d95, State: Running, Role: LEADER
12:12:59.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.988855  3951 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:12:59.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:12:59.989558  2133 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 reported cstate change: term changed from 8 to 9, leader changed from 968c62486e7148c4978383fb424be31b (127.1.230.129) to 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130). New cstate: current_term: 9 leader_uuid: "010544b667d64cc6b3fa372f3d439d95" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: UNKNOWN } } }
12:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.510877  3641 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 -> Peer 968c62486e7148c4978383fb424be31b (127.1.230.129:39309): Couldn't send request to peer 968c62486e7148c4978383fb424be31b. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.129:39309: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:13:00.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.554545  3887 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 9 FOLLOWER]: Refusing update from remote peer 010544b667d64cc6b3fa372f3d439d95: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
12:13:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.554939  3969 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
12:13:00.617 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:00.618 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:00.618 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2002), Traces: [0ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [2ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [2ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [22ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [23ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [23ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [41ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [42ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [44ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [44ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [82ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [84ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [84ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [179ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [182ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [183ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [185ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [185ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [379ms] refreshing cache from master, [380ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [382ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [383ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [384ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [385ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [899ms] refreshing cache from master, [900ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [901ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [903ms] sending RPC to server 968c62486e7148c4978383fb424be31b, [904ms] received response from server 968c62486e7148c4978383fb424be31b: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [905ms] delaying RPC due to: Network error: Failed to connect to peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309): Connection refused: /127.1.230.129:39309, [1999ms] refreshing cache from master, [2000ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [2001ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@406447331(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@236d613d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917) -> (continuation of Deferred@794873505 after retry RPC@1756923450) -> (continuation of Deferred@1765323101 after retry RPC@1167946518) -> (continuation of Deferred@1264950265 after retry RPC@1699632192), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f1cf264 -> wakeup thread Time-limited test -> (continuation of Deferred@1025610785 after retry RPC@900128307) -> (continuation of Deferred@477422310 after retry RPC@530636829) -> (continuation of Deferred@503016424 after retry RPC@885946917) -> (continuation of Deferred@794873505 after retry RPC@1756923450) -> (continuation of Deferred@1765323101 after retry RPC@1167946518) -> (continuation of Deferred@1264950265 after retry RPC@1699632192))) after lookup
12:13:00.619 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:13:00.620 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] Successfully connected to peer
12:13:00.634 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xba2207c5, L:/127.0.0.1:56818 - R:/127.1.230.130:42273] using TOKEN/null
12:13:00.642 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.129:39309
12:13:00.643 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.643532  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:13:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:39309
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34273
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:13:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:13:00.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.722962  3983 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:13:00.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.723136  3983 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:13:00.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.723165  3983 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:13:00.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.724604  3983 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:13:00.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.724658  3983 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.129
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.726132  3983 tablet_server_runner.cc:78] Tablet server non-default flags:
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.129:39309
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.129
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34273
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3983
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.129
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/logs
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:13:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:13:00.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:13:00.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:13:00.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.726323  3983 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:13:00.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.726507  3983 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:13:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.729067  3991 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.729094  3988 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:00.729372  3989 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.729524  3983 server_base.cc:1061] running on GCE node
12:13:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.729688  3983 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:13:00.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.729877  3983 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:13:00.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.731014  3983 hybrid_clock.cc:648] HybridClock initialized: now 1783512780730976 us; error 49 us; skew 500 ppm
12:13:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.732131  3983 webserver.cc:533] Webserver started at http://127.1.230.129:34273/ using document root <none> and password file <none>
12:13:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.732293  3983 fs_manager.cc:362] Metadata directory not provided
12:13:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.732333  3983 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:13:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.733464  3983 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.734051  3997 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.734230  3983 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.734283  3983 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "968c62486e7148c4978383fb424be31b"
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.734572  3983 fs_report.cc:389] FS layout report
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/wal
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/data
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:13:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:13:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:13:00.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.744454  3983 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:13:00.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.744712  3983 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:13:00.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.744796  3983 kserver.cc:163] Server-wide thread pool size limit: 3276
12:13:00.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.744961  3983 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:13:00.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.745347  4004 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:13:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.746253  3983 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:13:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.746299  3983 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:13:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.746323  3983 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:13:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.746892  3983 ts_tablet_manager.cc:616] Registered 1 tablets
12:13:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.746924  3983 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:13:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.747007  4004 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap starting.
12:13:00.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.755936  3983 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.129:39309
12:13:00.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.756078  4111 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.129:39309 every 8 connection(s)
12:13:00.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.756368  3983 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-0/data/info.pb
12:13:00.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.756578  4004 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Log is configured to *not* fsync() on all Append() calls
12:13:00.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.758193  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 3983
12:13:00.758 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:13:00.762 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:13:00.763 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.130:42273
12:13:00.763 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:13:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.763386  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 3624
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.766943  4004 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767089  4112 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767160  4112 heartbeater.cc:461] Registering TS with master...
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767172  4113 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767239  4113 heartbeater.cc:461] Registering TS with master...
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767359  4112 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:13:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767369  4113 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767956  4004 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Bootstrap complete.
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.767948  1989 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.768404  2133 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:13:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.768864  2133 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.129:54507
12:13:00.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.769064  4004 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent bootstrapping tablet: real 0.022s	user 0.015s	sys 0.000s
12:13:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.770037  4114 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:13:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.770099  4114 heartbeater.cc:461] Registering TS with master...
12:13:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.770174  4114 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:13:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.770509  2060 ts_manager.cc:194] Re-registered known tserver with Master: 968c62486e7148c4978383fb424be31b (127.1.230.129:39309)
12:13:00.771 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] handling channelInactive
12:13:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.770912  4004 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:00.771 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state READY due to: connection closed
12:13:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.771198  4004 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 968c62486e7148c4978383fb424be31b, State: Initialized, Role: FOLLOWER
12:13:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.771306  4004 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.771567  4004 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b: Time spent starting tablet: real 0.002s	user 0.003s	sys 0.000s
12:13:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:00.771606  4112 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:13:00.772 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:13:00.773 [DEBUG - Time-limited test] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:13:00.774 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.774 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.774 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.777 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:00.777 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:00.777 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [2ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [4ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1385025529(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@45767dbc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7687b9b5 -> wakeup thread Time-limited test)) after lookup
12:13:00.777 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:13:00.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.778 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.797 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:00.798 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:00.798 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [2ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [4ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [5ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [6ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [6ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1385025529(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@45767dbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7687b9b5 -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238))) after lookup
12:13:00.798 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:13:00.799 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.800 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.837 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:00.838 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:00.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [2ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [4ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [5ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [6ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [6ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [26ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [27ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1385025529(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@45767dbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7687b9b5 -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325))) after lookup
12:13:00.838 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:13:00.839 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.839 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.839 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.898 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:00.898 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:00.898 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=126), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [2ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [4ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [5ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [6ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [6ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [26ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [27ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [66ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [67ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [67ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [125ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1385025529(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@45767dbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325) -> (continuation of Deferred@1744144480 after retry RPC@200606321), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7687b9b5 -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325) -> (continuation of Deferred@1744144480 after retry RPC@200606321))) after lookup
12:13:00.899 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:13:00.900 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.900 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:00.900 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:01.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:01.338 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)[L],57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:01.338 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=565), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [2ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [4ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [5ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [6ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [6ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [26ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [27ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [66ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [67ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [67ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [125ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [127ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [128ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [128ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [563ms] refreshing cache from master, [563ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [565ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1385025529(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@45767dbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325) -> (continuation of Deferred@1744144480 after retry RPC@200606321) -> (continuation of Deferred@872985361 after retry RPC@569879330), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7687b9b5 -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325) -> (continuation of Deferred@1744144480 after retry RPC@200606321) -> (continuation of Deferred@872985361 after retry RPC@569879330))) after lookup
12:13:01.338 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] connecting to peer
12:13:01.340 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:01.340 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] cleaning up while in state CONNECTING due to: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:01.340 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273
12:13:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.143133  4125 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 010544b667d64cc6b3fa372f3d439d95)
12:13:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.143271  4125 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.143476  4125 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:13:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.144304  3819 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.147123  4066 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b" is_pre_election: true
12:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.147269  4066 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 8.
12:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.147496  3821 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b; no voters: 010544b667d64cc6b3fa372f3d439d95
12:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.147629  4125 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 9 FOLLOWER]: Leader pre-election won for term 10
12:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.147672  4125 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 010544b667d64cc6b3fa372f3d439d95)
12:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.147699  4125 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 9 FOLLOWER]: Advancing to term 10
12:13:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.148291  4125 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.148456  4125 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 10 election: Requested vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:13:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.148620  4066 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "57953137991b435ea05b795d05ae73c9" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b"
12:13:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.148718  4066 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 8 FOLLOWER]: Advancing to term 10
12:13:02.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.149039  3819 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111)
12:13:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.149936  4066 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57953137991b435ea05b795d05ae73c9 in term 10.
12:13:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.150158  3821 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b; no voters: 010544b667d64cc6b3fa372f3d439d95
12:13:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.150277  4125 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 10 FOLLOWER]: Leader election won for term 10
12:13:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.150477  4125 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 10 LEADER]: Becoming Leader. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Running, Role: LEADER
12:13:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.150718  4125 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:02.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.151582  2133 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 reported cstate change: term changed from 9 to 10, leader changed from 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130) to 57953137991b435ea05b795d05ae73c9 (127.1.230.131). New cstate: current_term: 10 leader_uuid: "57953137991b435ea05b795d05ae73c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: HEALTHY } } }
12:13:02.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.530383  4066 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 10 FOLLOWER]: Refusing update from remote peer 57953137991b435ea05b795d05ae73c9: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
12:13:02.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.530756  4125 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
12:13:02.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.533553  3819 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 -> Peer 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273): Couldn't send request to peer 010544b667d64cc6b3fa372f3d439d95. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.130:42273: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:13:02.658 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:02.658 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:02.659 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1886), Traces: [0ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [2ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [2ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [4ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [5ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [6ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [6ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [25ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [26ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [27ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [28ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [65ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [66ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [67ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [67ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [123ms] refreshing cache from master, [123ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [125ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [127ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [128ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [128ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [563ms] refreshing cache from master, [563ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [565ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [566ms] sending RPC to server 010544b667d64cc6b3fa372f3d439d95, [568ms] received response from server 010544b667d64cc6b3fa372f3d439d95: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [568ms] delaying RPC due to: Network error: Failed to connect to peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273): Connection refused: /127.1.230.130:42273, [1883ms] refreshing cache from master, [1884ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [1886ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@1385025529(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@45767dbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325) -> (continuation of Deferred@1744144480 after retry RPC@200606321) -> (continuation of Deferred@872985361 after retry RPC@569879330) -> (continuation of Deferred@1476663870 after retry RPC@1144390823), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7687b9b5 -> wakeup thread Time-limited test -> (continuation of Deferred@147323954 after retry RPC@1137092238) -> (continuation of Deferred@1061105474 after retry RPC@1236995325) -> (continuation of Deferred@1744144480 after retry RPC@200606321) -> (continuation of Deferred@872985361 after retry RPC@569879330) -> (continuation of Deferred@1476663870 after retry RPC@1144390823))) after lookup
12:13:02.659 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:02.660 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] Successfully connected to peer
12:13:02.688 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0xcf4bfe78, L:/127.0.0.1:53708 - R:/127.1.230.131:38541] using TOKEN/null
12:13:02.698 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.130:42273
12:13:02.698 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.699146  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:13:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:42273
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46119
12:13:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:13:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:13:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:13:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:13:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:13:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.784881  4153 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:13:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.785077  4153 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:13:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.785104  4153 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:13:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.786798  4153 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:13:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.786861  4153 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.130
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.788542  4153 tablet_server_runner.cc:78] Tablet server non-default flags:
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.130:42273
12:13:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.130
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46119
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4153
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.130
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/logs
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.788816  4153 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:13:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.789034  4153 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:13:02.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.791738  4161 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:02.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.791774  4158 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:02.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:02.791798  4159 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:02.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.791875  4153 server_base.cc:1061] running on GCE node
12:13:02.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.792327  4153 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:13:02.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.792522  4153 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:13:02.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.793728  4153 hybrid_clock.cc:648] HybridClock initialized: now 1783512782793696 us; error 43 us; skew 500 ppm
12:13:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.795146  4153 webserver.cc:533] Webserver started at http://127.1.230.130:46119/ using document root <none> and password file <none>
12:13:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.795403  4153 fs_manager.cc:362] Metadata directory not provided
12:13:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.795459  4153 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:13:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.796576  4153 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
12:13:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.797164  4167 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:13:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.797354  4153 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
12:13:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.797430  4153 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:13:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "010544b667d64cc6b3fa372f3d439d95"
12:13:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:13:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.797729  4153 fs_report.cc:389] FS layout report
12:13:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/wal
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/data
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:13:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:13:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.807839  4153 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:13:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.808187  4153 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:13:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.808362  4153 kserver.cc:163] Server-wide thread pool size limit: 3276
12:13:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.808619  4153 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:13:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.809152  4174 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:13:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.809989  4153 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:13:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.810052  4153 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:13:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.810088  4153 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:13:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.810657  4153 ts_tablet_manager.cc:616] Registered 1 tablets
12:13:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.810760  4153 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
12:13:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.810834  4174 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap starting.
12:13:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.817731  4153 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.130:42273
12:13:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.817891  4281 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.130:42273 every 8 connection(s)
12:13:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.818141  4153 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-1/data/info.pb
12:13:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.821832  4174 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Log is configured to *not* fsync() on all Append() calls
12:13:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.823990  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 4153
12:13:02.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:13:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.826001  4282 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:13:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.826116  4282 heartbeater.cc:461] Registering TS with master...
12:13:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.826327  4282 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:13:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.826354  4284 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:13:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.826422  4284 heartbeater.cc:461] Registering TS with master...
12:13:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.826527  4284 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:13:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.826987  2132 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:13:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.827245  2060 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:13:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.827538  2132 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.130:42877
12:13:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.828428  4174 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:13:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.828529  4283 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:13:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.828593  4283 heartbeater.cc:461] Registering TS with master...
12:13:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.828680  4283 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:13:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.828763  4174 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Bootstrap complete.
12:13:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.828997  1989 ts_manager.cc:194] Re-registered known tserver with Master: 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273)
12:13:02.829 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:13:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.830001  4174 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent bootstrapping tablet: real 0.019s	user 0.010s	sys 0.006s
12:13:02.830 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.230.131:38541
12:13:02.830 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:13:02.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.830892  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 3804
12:13:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.832017  4174 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.832226  4174 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 010544b667d64cc6b3fa372f3d439d95, State: Initialized, Role: FOLLOWER
12:13:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.832352  4174 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.832638  4282 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:13:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:02.832628  4174 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95: Time spent starting tablet: real 0.003s	user 0.002s	sys 0.000s
12:13:02.840 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] handling channelInactive
12:13:02.840 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state READY due to: connection closed
12:13:02.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:13:02.841 [DEBUG - Time-limited test] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:02.842 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.842 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.842 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.857 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:02.858 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:02.858 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [1ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@897649774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@41e62545 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@68b829f4 -> wakeup thread Time-limited test)) after lookup
12:13:02.859 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:02.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.860 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.860 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.877 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:02.877 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:02.877 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [1ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [17ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [19ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [19ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@897649774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@41e62545 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@68b829f4 -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827))) after lookup
12:13:02.878 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:02.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.879 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.897 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:02.897 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:02.897 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [1ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [17ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [19ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [19ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [37ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [38ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [53ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [55ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@897649774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@41e62545 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@68b829f4 -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039))) after lookup
12:13:02.897 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:02.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.899 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.958 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:02.958 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:02.958 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=117), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [1ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [17ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [19ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [19ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [37ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [38ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [53ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [55ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [56ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [58ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [114ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [116ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@897649774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@41e62545 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@68b829f4 -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454))) after lookup
12:13:02.959 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:02.959 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.959 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:02.960 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:03.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:03.337 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:03.337 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=496), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [1ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [17ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [19ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [19ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [37ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [38ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [53ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [55ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [56ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [58ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [114ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [116ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [117ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [118ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [119ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [493ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [495ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@897649774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@41e62545 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454) -> (continuation of Deferred@1465809606 after retry RPC@62619060), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@68b829f4 -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454) -> (continuation of Deferred@1465809606 after retry RPC@62619060))) after lookup
12:13:03.338 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:03.339 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:03.339 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:03.339 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:03.557 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:03.557 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)[L],968c62486e7148c4978383fb424be31b(127.1.230.129:39309)]}]
12:13:03.557 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=716), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [1ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [17ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [19ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [19ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [37ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [38ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [53ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [55ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [56ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [58ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [114ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [116ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [117ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [118ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [119ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [493ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [495ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [497ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [498ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [498ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [713ms] refreshing cache from master, [714ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [715ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@897649774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@41e62545 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454) -> (continuation of Deferred@1465809606 after retry RPC@62619060) -> (continuation of Deferred@1418593245 after retry RPC@201364005), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@68b829f4 -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454) -> (continuation of Deferred@1465809606 after retry RPC@62619060) -> (continuation of Deferred@1418593245 after retry RPC@201364005))) after lookup
12:13:03.558 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] connecting to peer
12:13:03.559 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:03.559 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:03.559 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541) for tablet c68a9df270ca4b13b47fb31a1b9431a7: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541
12:13:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.127254  4295 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
12:13:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.127421  4295 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.127784  4295 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 968c62486e7148c4978383fb424be31b (127.1.230.129:39309), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:13:04.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.128262  4171 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:13:04.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.128731  4171 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:13:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.131349  4066 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "010544b667d64cc6b3fa372f3d439d95" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "968c62486e7148c4978383fb424be31b" is_pre_election: true
12:13:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.131620  4170 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95; no voters: 57953137991b435ea05b795d05ae73c9, 968c62486e7148c4978383fb424be31b
12:13:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.131755  4295 raft_consensus.cc:2749] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
12:13:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.197923  4300 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:13:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.198035  4300 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.198371  4300 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:13:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.198978  4001 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:13:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.199491  4001 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:13:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.201534  4236 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95" is_pre_election: true
12:13:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.201659  4236 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 9.
12:13:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.201910  3998 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 57953137991b435ea05b795d05ae73c9
12:13:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.202039  4300 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 10 FOLLOWER]: Leader pre-election won for term 11
12:13:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.202122  4300 raft_consensus.cc:493] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 10 FOLLOWER]: Starting leader election (detected failure of leader 57953137991b435ea05b795d05ae73c9)
12:13:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.202152  4300 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 10 FOLLOWER]: Advancing to term 11
12:13:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.202836  4300 raft_consensus.cc:515] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.202991  4300 leader_election.cc:290] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 11 election: Requested vote from peers 010544b667d64cc6b3fa372f3d439d95 (127.1.230.130:42273), 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:13:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.203164  4236 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c68a9df270ca4b13b47fb31a1b9431a7" candidate_uuid: "968c62486e7148c4978383fb424be31b" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "010544b667d64cc6b3fa372f3d439d95"
12:13:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.203258  4236 raft_consensus.cc:3060] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 9 FOLLOWER]: Advancing to term 11
12:13:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.203504  4001 leader_election.cc:336] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111)
12:13:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.204252  4236 raft_consensus.cc:2468] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 968c62486e7148c4978383fb424be31b in term 11.
12:13:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.204440  3998 leader_election.cc:304] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 010544b667d64cc6b3fa372f3d439d95, 968c62486e7148c4978383fb424be31b; no voters: 57953137991b435ea05b795d05ae73c9
12:13:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.204517  4300 raft_consensus.cc:2804] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 11 FOLLOWER]: Leader election won for term 11
12:13:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.204650  4300 raft_consensus.cc:697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [term 11 LEADER]: Becoming Leader. State: Replica: 968c62486e7148c4978383fb424be31b, State: Running, Role: LEADER
12:13:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.204717  4300 consensus_queue.cc:237] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.205332  2132 catalog_manager.cc:5697] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b reported cstate change: term changed from 10 to 11, leader changed from 57953137991b435ea05b795d05ae73c9 (127.1.230.131) to 968c62486e7148c4978383fb424be31b (127.1.230.129). New cstate: current_term: 11 leader_uuid: "968c62486e7148c4978383fb424be31b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } health_report { overall_health: UNKNOWN } } }
12:13:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.669734  4001 consensus_peers.cc:597] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b -> Peer 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541): Couldn't send request to peer 57953137991b435ea05b795d05ae73c9. Status: Network error: Client connection negotiation failed: client connection to 127.1.230.131:38541: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
12:13:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.766297  4236 raft_consensus.cc:1275] T c68a9df270ca4b13b47fb31a1b9431a7 P 010544b667d64cc6b3fa372f3d439d95 [term 11 FOLLOWER]: Refusing update from remote peer 968c62486e7148c4978383fb424be31b: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
12:13:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.766702  4305 consensus_queue.cc:1048] T c68a9df270ca4b13b47fb31a1b9431a7 P 968c62486e7148c4978383fb424be31b [LEADER]: Connected to new peer: Peer: permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
12:13:04.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet c68a9df270ca4b13b47fb31a1b9431a7 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859' with partition [<start>, <end>)
12:13:04.838 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c68a9df270ca4b13b47fb31a1b9431a7@[010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273),57953137991b435ea05b795d05ae73c9(127.1.230.131:38541),968c62486e7148c4978383fb424be31b(127.1.230.129:39309)[L]]}]
12:13:04.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c68a9df270ca4b13b47fb31a1b9431a7, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1997), Traces: [0ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [1ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [16ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [17ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [19ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [19ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [36ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [37ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [38ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [38ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [53ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [55ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [56ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [58ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [58ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [114ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [116ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [117ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [118ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [119ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [493ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [495ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [497ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [498ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [498ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [713ms] refreshing cache from master, [714ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [715ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, [716ms] sending RPC to server 57953137991b435ea05b795d05ae73c9, [718ms] received response from server 57953137991b435ea05b795d05ae73c9: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [718ms] delaying RPC due to: Network error: Failed to connect to peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541): Connection refused: /127.1.230.131:38541, [1994ms] refreshing cache from master, [1994ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.230.188:43411, [1996ms] Sub RPC GetTableLocations: received response from server master-127.1.230.188:43411: OK, deferred=Deferred@897649774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$237/0x00007f4a7436bed0@41e62545 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454) -> (continuation of Deferred@1465809606 after retry RPC@62619060) -> (continuation of Deferred@1418593245 after retry RPC@201364005) -> (continuation of Deferred@1070786544 after retry RPC@927714197), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@68b829f4 -> wakeup thread Time-limited test -> (continuation of Deferred@151737055 after retry RPC@550845827) -> (continuation of Deferred@850494399 after retry RPC@998345039) -> (continuation of Deferred@2062437077 after retry RPC@1301446454) -> (continuation of Deferred@1465809606 after retry RPC@62619060) -> (continuation of Deferred@1418593245 after retry RPC@201364005) -> (continuation of Deferred@1070786544 after retry RPC@927714197))) after lookup
12:13:04.839 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] connecting to peer
12:13:04.840 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] Successfully connected to peer
12:13:04.857 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0xb9965e2d, L:/127.0.0.1:54804 - R:/127.1.230.129:39309] using TOKEN/null
12:13:04.867 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.230.131:38541
12:13:04.867 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

12:13:04.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.867765  1946 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSxpMW8/build/release/bin/kudu
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:13:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:13:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
12:13:04.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.944820  4328 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
12:13:04.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.944981  4328 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
12:13:04.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.945000  4328 flags.cc:432] Enabled unsafe flag: --never_fsync=true
12:13:04.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.946403  4328 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
12:13:04.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.946449  4328 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.230.131
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.947973  4328 tablet_server_runner.cc:78] Tablet server non-default flags:
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.230.148:35549
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.230.131:38541
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.230.131
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33199
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.230.190:40253,127.1.230.189:40567,127.1.230.188:43411
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4328
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.230.131
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/logs
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8aadabe0236da0a312e7e3764fada9dc81e70c40
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 11:49:00 UTC on e7f111948823
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16855
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.948161  4328 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
12:13:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.948342  4328 file_cache.cc:492] Constructed file cache file cache with capacity 419430
12:13:04.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.950742  4336 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.950784  4333 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 12:13:04.950728  4334 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
12:13:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.951128  4328 server_base.cc:1061] running on GCE node
12:13:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.951277  4328 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
12:13:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.951432  4328 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
12:13:04.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.952558  4328 hybrid_clock.cc:648] HybridClock initialized: now 1783512784952538 us; error 36 us; skew 500 ppm
12:13:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.953639  4328 webserver.cc:533] Webserver started at http://127.1.230.131:33199/ using document root <none> and password file <none>
12:13:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.953833  4328 fs_manager.cc:362] Metadata directory not provided
12:13:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.953893  4328 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
12:13:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.955395  4328 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.002s	sys 0.000s
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.955969  4342 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.956135  4328 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.956187  4328 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data,/tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57953137991b435ea05b795d05ae73c9"
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 12:12:34 on dist-test-slave-dp7z"
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.956429  4328 fs_report.cc:389] FS layout report
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/wal
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/data
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
12:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
12:13:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.970019  4328 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
12:13:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.970273  4328 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
12:13:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.970366  4328 kserver.cc:163] Server-wide thread pool size limit: 3276
12:13:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.970525  4328 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
12:13:04.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.971076  4349 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
12:13:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.971921  4328 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
12:13:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.971976  4328 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
12:13:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.972010  4328 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
12:13:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.972616  4328 ts_tablet_manager.cc:616] Registered 1 tablets
12:13:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.972666  4328 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
12:13:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.972704  4349 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap starting.
12:13:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.979836  4328 rpc_server.cc:307] RPC server started. Bound to: 127.1.230.131:38541
12:13:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.979925  4456 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.230.131:38541 every 8 connection(s)
12:13:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.980264  4328 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskSxpMW8/test-tmp/mini-kudu-cluster2407809955293230683/ts-2/data/info.pb
12:13:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.981740  4349 log.cc:826] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Log is configured to *not* fsync() on all Append() calls
12:13:04.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.982086  1946 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskSxpMW8/build/release/bin/kudu as pid 4328
12:13:04.982 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
12:13:04.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.986821  4457 heartbeater.cc:344] Connected to a master server at 127.1.230.188:43411
12:13:04.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.986907  4457 heartbeater.cc:461] Registering TS with master...
12:13:04.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.987078  4457 heartbeater.cc:507] Master 127.1.230.188:43411 requested a full tablet report, sending...
12:13:04.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.987537  2132 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:13:04.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.988041  2132 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.230.131:49625
12:13:04.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.988740  4349 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
12:13:04.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.989038  4349 tablet_bootstrap.cc:492] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Bootstrap complete.
12:13:04.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.989838  4458 heartbeater.cc:344] Connected to a master server at 127.1.230.190:40253
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.989903  4458 heartbeater.cc:461] Registering TS with master...
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.989960  4458 heartbeater.cc:507] Master 127.1.230.190:40253 requested a full tablet report, sending...
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.990156  4459 heartbeater.cc:344] Connected to a master server at 127.1.230.189:40567
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.990204  4459 heartbeater.cc:461] Registering TS with master...
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.990182  4349 ts_tablet_manager.cc:1403] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent bootstrapping tablet: real 0.018s	user 0.012s	sys 0.000s
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.990245  4459 heartbeater.cc:507] Master 127.1.230.189:40567 requested a full tablet report, sending...
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.990300  1989 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:13:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.990610  2061 ts_manager.cc:194] Re-registered known tserver with Master: 57953137991b435ea05b795d05ae73c9 (127.1.230.131:38541)
12:13:04.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.991312  4349 raft_consensus.cc:359] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:04.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.991572  4349 raft_consensus.cc:740] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57953137991b435ea05b795d05ae73c9, State: Initialized, Role: FOLLOWER
12:13:04.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.991676  4349 consensus_queue.cc:260] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "968c62486e7148c4978383fb424be31b" member_type: VOTER last_known_addr { host: "127.1.230.129" port: 39309 } } peers { permanent_uuid: "010544b667d64cc6b3fa372f3d439d95" member_type: VOTER last_known_addr { host: "127.1.230.130" port: 42273 } } peers { permanent_uuid: "57953137991b435ea05b795d05ae73c9" member_type: VOTER last_known_addr { host: "127.1.230.131" port: 38541 } }
12:13:04.991 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:13:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.991926  4349 ts_tablet_manager.cc:1434] T c68a9df270ca4b13b47fb31a1b9431a7 P 57953137991b435ea05b795d05ae73c9: Time spent starting tablet: real 0.002s	user 0.001s	sys 0.003s
12:13:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:04.992405  4457 heartbeater.cc:499] Master 127.1.230.188:43411 was elected leader, sending a full tablet report...
12:13:04.994 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783512754859, tablet=c68a9df270ca4b13b47fb31a1b9431a7, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
12:13:04.999 [DEBUG - main] (Connection.java:575) [peer 010544b667d64cc6b3fa372f3d439d95(127.1.230.130:42273)] disconnecting while in state TERMINATED
12:13:05.000 [DEBUG - main] (Connection.java:575) [peer 57953137991b435ea05b795d05ae73c9(127.1.230.131:38541)] disconnecting while in state TERMINATED
12:13:05.001 [DEBUG - main] (Connection.java:575) [peer master-127.1.230.189:40567(127.1.230.189:40567)] disconnecting while in state READY
12:13:05.001 [DEBUG - main] (Connection.java:575) [peer master-127.1.230.190:40253(127.1.230.190:40253)] disconnecting while in state READY
12:13:05.001 [DEBUG - main] (Connection.java:575) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] disconnecting while in state READY
12:13:05.002 [DEBUG - main] (Connection.java:575) [peer master-127.1.230.188:43411(127.1.230.188:43411)] disconnecting while in state READY
12:13:05.004 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] handling channelInactive
12:13:05.004 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.230.190:40253(127.1.230.190:40253)] handling channelInactive
12:13:05.004 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
12:13:05.004 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 968c62486e7148c4978383fb424be31b(127.1.230.129:39309)] cleaning up while in state READY due to: connection closed
12:13:05.004 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.230.190:40253(127.1.230.190:40253)] cleaning up while in state READY due to: connection closed
12:13:05.005 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.1.230.188:43411(127.1.230.188:43411)] handling channelInactive
12:13:05.005 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.1.230.188:43411(127.1.230.188:43411)] cleaning up while in state READY due to: connection closed
12:13:05.005 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.230.189:40567(127.1.230.189:40567)] handling channelInactive
12:13:05.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.230.189:40567(127.1.230.189:40567)] cleaning up while in state READY due to: connection closed
12:13:05.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:05.008610  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 3983
12:13:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:05.013845  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 4153
12:13:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:05.018874  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 4328
12:13:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:05.023758  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 1958
12:13:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:05.029150  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 2029
12:13:05.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 12:13:05.033962  1946 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskSxpMW8/build/release/bin/kudu with pid 2101
12:13:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T12:13:05Z chronyd exiting

Time: 32.384

OK (1 test)

