Version in base suite: 26.4.18-0+deb12u1 Base version: galera-4_26.4.18-0+deb12u1 Target version: galera-4_26.4.20-0+deb12u1 Base file: /srv/ftp-master.debian.org/ftp/pool/main/g/galera-4/galera-4_26.4.18-0+deb12u1.dsc Target file: /srv/ftp-master.debian.org/policy/pool/main/g/galera-4/galera-4_26.4.20-0+deb12u1.dsc .jenkins/aws-galera-4-fullbuild.groovy | 92 ++ GALERA_GIT_REVISION | 2 GALERA_REVISION | 2 GALERA_VERSION | 2 SConstruct | 2 debian/changelog | 9 debian/gbp.conf | 54 + debian/salsa-ci.yml | 8 galera/src/galera-sym.map | 1 galera/src/gcs_action_source.cpp | 2 galera/src/ist.cpp | 24 galera/src/ist.hpp | 11 galera/src/ist_proto.hpp | 3 galera/src/key_data.hpp | 7 galera/src/key_set.cpp | 160 +--- galera/src/key_set.hpp | 1 galera/src/mapped_buffer.cpp | 17 galera/src/replicator_smm.cpp | 28 galera/src/replicator_smm.hpp | 28 galera/src/replicator_str.cpp | 12 galera/src/saved_state.cpp | 2 galera/src/trx_handle.hpp | 15 galera/src/wsrep_provider.cpp | 26 galera/tests/certification_check.cpp | 1008 +++++++++++++++++++++++++++++-- galera/tests/ist_check.cpp | 12 galera/tests/key_set_check.cpp | 561 +++++++++++++++-- galera/tests/test_key.hpp | 36 - galera/tests/write_set_ng_check.cpp | 19 galerautils/src/gu_asio.cpp | 11 galerautils/src/gu_asio.hpp | 13 galerautils/src/gu_asio_datagram.cpp | 10 galerautils/src/gu_asio_socket_util.hpp | 16 galerautils/src/gu_asio_stream_react.cpp | 205 +++--- galerautils/src/gu_asio_stream_react.hpp | 17 galerautils/src/gu_datetime.cpp | 8 galerautils/src/gu_fdesc.cpp | 17 galerautils/src/gu_fifo.c | 10 galerautils/src/gu_init.c | 2 galerautils/src/gu_lock.hpp | 2 galerautils/src/gu_log.c | 9 galerautils/src/gu_log.h | 27 galerautils/src/gu_mmap.cpp | 12 galerautils/src/gu_mutex.hpp | 4 galerautils/src/gu_resolver.cpp | 6 galerautils/src/gu_thread.cpp | 5 galerautils/src/gu_throw.hpp | 37 + galerautils/src/gu_to.c | 70 +- galerautils/tests/crc32c_bench.cpp | 10 galerautils/tests/gu_asio_test.cpp | 654 +++++++++++--------- garb/garb_main.cpp | 18 garb/garb_recv_loop.cpp | 10 gcache/src/gcache_page_store.cpp | 11 gcomm/src/asio_tcp.cpp | 35 - gcomm/src/asio_tcp.hpp | 4 gcomm/src/evs_proto.cpp | 5 gcomm/src/gcomm/protolay.hpp | 1 gcomm/src/gmcast.cpp | 6 gcomm/src/gmcast_proto.cpp | 8 gcomm/src/pc_proto.cpp | 32 gcs/src/CMakeLists.txt | 1 gcs/src/SConscript | 1 gcs/src/gcs.cpp | 118 ++- gcs/src/gcs_act_proto.cpp | 2 gcs/src/gcs_act_proto.hpp | 4 gcs/src/gcs_core.cpp | 72 +- gcs/src/gcs_defrag.cpp | 36 - gcs/src/gcs_dummy.cpp | 6 gcs/src/gcs_error.cpp | 35 + gcs/src/gcs_error.hpp | 53 + gcs/src/gcs_gcomm.cpp | 6 gcs/src/gcs_group.cpp | 84 +- gcs/src/gcs_node.cpp | 20 gcs/src/gcs_node.hpp | 11 gcs/src/gcs_sm.cpp | 2 gcs/src/gcs_state_msg.cpp | 7 gcs/src/unit_tests/CMakeLists.txt | 1 gcs/src/unit_tests/SConscript | 1 gcs/src/unit_tests/gcs_core_test.cpp | 2 gcs/src/unit_tests/gcs_test_utils.cpp | 4 scripts/packages/codership-galera.spec | 2 wsrep/src/wsrep_api.h | 2 wsrep/src/wsrep_loader.c | 9 wsrep/src/wsrep_node_isolation.h | 70 ++ 83 files changed, 2990 insertions(+), 978 deletions(-) diff -Nru galera-4-26.4.18/.jenkins/aws-galera-4-fullbuild.groovy galera-4-26.4.20/.jenkins/aws-galera-4-fullbuild.groovy --- galera-4-26.4.18/.jenkins/aws-galera-4-fullbuild.groovy 1970-01-01 00:00:00.000000000 +0000 +++ galera-4-26.4.20/.jenkins/aws-galera-4-fullbuild.groovy 2024-07-30 05:28:41.000000000 +0000 @@ -0,0 +1,92 @@ + +pipeline { + + agent none + + stages { + + stage ('Build sourcetar') { + steps { + script { + def sourceJob = build job: 'aws-galera-4-sourcetar', wait: true, + parameters: [ + string(name: 'GIT_TARGET', value: env.GIT_TARGET ), + booleanParam( name: 'HOTFIX_BUILD', value: env.HOTFIX_BUILD) + ] + env.SRCTAR_JOB = sourceJob.getNumber().toString() + } + } + } + + stage ('Build binary packages') { + + parallel { + stage ('Build bintar') { + steps { + script { + def bintarJob = build job: 'aws-galera-4-bintar', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.SRCTAR_JOB )] + env.BINTAR_JOB = bintarJob.getNumber().toString() + } + } + } + stage ('Build rpm packages') { + steps { + script { + def rpmJob = build job: 'aws-galera-4-rpm-packages', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.SRCTAR_JOB )] + env.RPM_JOB = rpmJob.getNumber().toString() + } + } + } + stage ('Build deb packages') { + steps { + script { + def debJob = build job: 'aws-galera-4-deb-packages', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.SRCTAR_JOB )] + env.DEB_JOB = debJob.getNumber().toString() + } + } + } + } // parallel + + } // Build binary packages + + stage ('Run tests') { + parallel { + stage('Run bintar test') { + steps { + build job: 'run-galera-4-release-test', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.BINTAR_JOB )] + } + } + stage ('Run RPM test') { + steps { + build job: 'run-galera-4-rpm-test', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.RPM_JOB )] + } + } + stage ('Run DEB test') { + steps { + build job: 'run-galera-4-deb-test', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.DEB_JOB )] + } + } + stage ('Run SST RPM test') { + steps { + build job: 'run-galera-4-systemd-sst-rpm-test', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.RPM_JOB )] + } + } + stage ('Run SST DEB test') { + steps { + build job: 'run-galera-4-systemd-sst-deb-test', wait: true, + parameters: [string(name: 'BUILD_SELECTOR', value: env.DEB_JOB )] + } + } + } // parallel + } + + } // stages + +} diff -Nru galera-4-26.4.18/GALERA_GIT_REVISION galera-4-26.4.20/GALERA_GIT_REVISION --- galera-4-26.4.18/GALERA_GIT_REVISION 2024-03-22 11:54:22.000000000 +0000 +++ galera-4-26.4.20/GALERA_GIT_REVISION 2024-07-30 05:28:42.000000000 +0000 @@ -1 +1 @@ -0bc393fb \ No newline at end of file +fed86127 \ No newline at end of file diff -Nru galera-4-26.4.18/GALERA_REVISION galera-4-26.4.20/GALERA_REVISION --- galera-4-26.4.18/GALERA_REVISION 2024-03-22 11:54:22.000000000 +0000 +++ galera-4-26.4.20/GALERA_REVISION 2024-07-30 05:28:42.000000000 +0000 @@ -1 +1 @@ -0bc393fb \ No newline at end of file +fed86127 \ No newline at end of file diff -Nru galera-4-26.4.18/GALERA_VERSION galera-4-26.4.20/GALERA_VERSION --- galera-4-26.4.18/GALERA_VERSION 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/GALERA_VERSION 2024-07-30 05:28:41.000000000 +0000 @@ -1,4 +1,4 @@ GALERA_VERSION_WSREP_API=26 GALERA_VERSION_MAJOR=4 -GALERA_VERSION_MINOR=18 +GALERA_VERSION_MINOR=20 GALERA_VERSION_EXTRA= diff -Nru galera-4-26.4.18/SConstruct galera-4-26.4.20/SConstruct --- galera-4-26.4.18/SConstruct 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/SConstruct 2024-07-30 05:28:41.000000000 +0000 @@ -163,7 +163,7 @@ install = ARGUMENTS.get('install', None) version_script = int(ARGUMENTS.get('version_script', 1)) -GALERA_VER = ARGUMENTS.get('version', '4.18') +GALERA_VER = ARGUMENTS.get('version', '4.20') GALERA_REV = ARGUMENTS.get('revno', 'XXXX') # Attempt to read from file if not given diff -Nru galera-4-26.4.18/debian/changelog galera-4-26.4.20/debian/changelog --- galera-4-26.4.18/debian/changelog 2024-04-22 04:52:19.000000000 +0000 +++ galera-4-26.4.20/debian/changelog 2024-08-31 07:26:53.000000000 +0000 @@ -1,3 +1,12 @@ +galera-4 (26.4.20-0+deb12u1) bookworm; urgency=medium + + * New upstream release 26.4.20. Includes multiple bug fixes, see + https://github.com/codership/documentation/blob/master/release-notes/release-notes-galera-26.4.20.txt + https://github.com/codership/documentation/blob/master/release-notes/release-notes-galera-26.4.19.txt + * Adopt gbp.conf from 'debian/latest' but adopt it for 'debian/12-bookworm' + + -- Otto Kekäläinen Sat, 31 Aug 2024 00:26:53 -0700 + galera-4 (26.4.18-0+deb12u1) bookworm; urgency=medium * Switch to upstream aware DEP-14 branch structure in gbp.conf diff -Nru galera-4-26.4.18/debian/gbp.conf galera-4-26.4.20/debian/gbp.conf --- galera-4-26.4.18/debian/gbp.conf 2024-04-22 04:52:19.000000000 +0000 +++ galera-4-26.4.20/debian/gbp.conf 2024-08-31 07:26:53.000000000 +0000 @@ -1,19 +1,51 @@ [DEFAULT] -# Ignore requirement to use branch name 'master' to make it easier -# for contributors to work with feature and bugfix branches -ignore-branch = True +# DEP14 format +debian-branch = debian/12-bookworm +upstream-branch = upstream/latest +upstream-tag = upstream/%(version)s + +# Ensure we always target Debian on Debian branches +dch-opt = --vendor=debian # Always use pristine tar pristine-tar = True -# Sign tags +# This git repository also hosts the actual upstream tags and main branch '4.x'. +# Configure the upstream tag format below, so that `gbp import-orig` will run +# correctly, and link tarball import branch (`upstream/latest`) with the +# equivalent upstream release tag, showing a complete audit trail of what +# upstream released and what was imported into Debian. +upstream-vcs-tag = release_%(version%~%.)s + +# Check that upstream signed git tags (options: auto|on|off) +upstream-signatures = on + +# Ensure the Debian maintainer signs git tags automatically sign-tags = True -# DEP-14 format -debian-branch = debian/bookworm +# Use author information from git +git-author = True + +# Ease dropping / adding patches +patch-numbers = False + +# Ensure changes are *not* listed in multiple sections with the same "[ Author ]" +multimaint-merge = True + +# Lax requirement to use branch name 'debian/latest' so that git-buildpackage +# will always build using the currently checked out branch as the Debian branch. +# This makes it easier for contributors to work with feature and bugfix +# branches. +ignore-branch = True + +# Automatically open a new changelog entry about the new upstream release, but +# do not commit it, as the 'gbp dch' still needs to run and list all commits +# based on when the debian/changelog last was updated in a git commit +postimport = dch -v %(version)s "New upstream release" + +# Ensure a human always reviews all the debian/changelog entries +spawn-editor=always -# Upstream branch continues to be the default 'debian/upstream' and is used for -# upstream tarball imports, with each import tagged 'upstream/%(version)s'. -# Additionally the repository hosts also the actual upstream main branch '4.x' -# and upstream tags so that git-buildpackage can link all of them together. -upstream-vcs-tag=release_%(version)s +# No need to confirm package name or version at any time, git-buildpackage +# always gets it right +interactive = False diff -Nru galera-4-26.4.18/debian/salsa-ci.yml galera-4-26.4.20/debian/salsa-ci.yml --- galera-4-26.4.18/debian/salsa-ci.yml 2024-04-22 04:52:19.000000000 +0000 +++ galera-4-26.4.20/debian/salsa-ci.yml 2024-08-31 07:26:53.000000000 +0000 @@ -5,10 +5,12 @@ - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml variables: - SALSA_CI_DISABLE_MISSING_BREAKS: 0 - SALSA_CI_DISABLE_RC_BUGS: 0 SALSA_CI_DISABLE_BLHC: 1 - RELEASE: bookworm + SALSA_CI_DISABLE_PIUPARTS: 1 + SALSA_CI_DISABLE_REPROTEST: 1 + SALSA_CI_DISABLE_BUILD_PACKAGE_ALL: 1 + SALSA_CI_DISABLE_BUILD_PACKAGE_ANY: 1 + SALSA_CI_DISABLE_BUILD_PACKAGE_I386: 1 stages: - provisioning diff -Nru galera-4-26.4.18/galera/src/galera-sym.map galera-4-26.4.20/galera/src/galera-sym.map --- galera-4-26.4.18/galera/src/galera-sym.map 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/galera-sym.map 2024-07-30 05:28:41.000000000 +0000 @@ -7,5 +7,6 @@ wsrep_deinit_event_service_v1; wsrep_init_config_service_v1; wsrep_deinit_config_service_v1; + wsrep_node_isolation_mode_set_v1; local: *; }; diff -Nru galera-4-26.4.18/galera/src/gcs_action_source.cpp galera-4-26.4.20/galera/src/gcs_action_source.cpp --- galera-4-26.4.18/galera/src/gcs_action_source.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/gcs_action_source.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -190,6 +190,8 @@ } else if (rc > 0 && skip) { + Release release(act, gcache_); + replicator_.cancel_seqnos(act.seqno_l, act.seqno_g); } else diff -Nru galera-4-26.4.18/galera/src/ist.cpp galera-4-26.4.20/galera/src/ist.cpp --- galera-4-26.4.18/galera/src/ist.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/ist.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -341,7 +341,6 @@ return recv_addr_; } - void galera::ist::Receiver::run() { auto socket(acceptor_->accept()); @@ -351,6 +350,7 @@ gu::Progress* progress(NULL); int ec(0); + std::ostringstream error_os; try { @@ -396,9 +396,8 @@ assert(!progress); if (act.seqno_g > first_seqno_) { - log_error - << "IST started with wrong seqno: " << act.seqno_g - << ", expected <= " << first_seqno_; + error_os << "IST started with wrong seqno: " << act.seqno_g + << ", expected <= " << first_seqno_; ec = EINVAL; goto err; } @@ -424,8 +423,8 @@ if (act.seqno_g != current_seqno_) { - log_error << "Unexpected action seqno: " << act.seqno_g - << " expected: " << current_seqno_; + error_os << "Unexpected action seqno: " << act.seqno_g + << " expected: " << current_seqno_; ec = EINVAL; goto err; } @@ -488,7 +487,7 @@ ec = e.get_errno(); if (ec != EINTR) { - log_error << "got exception while reading IST stream: " << e.what(); + error_os << "got exception while reading IST stream: " << e.what(); } } @@ -498,17 +497,18 @@ socket->close(); running_ = false; - if (last_seqno_ > 0 && ec != EINTR && current_seqno_ < last_seqno_) + if (last_seqno_ > 0 && ec != EINTR && current_seqno_ < last_seqno_ && + error_os.tellp() == 0) { - log_error << "IST didn't contain all write sets, expected last: " - << last_seqno_ << " last received: " << current_seqno_; + error_os << "IST didn't contain all write sets, expected last: " + << last_seqno_ << " last received: " << current_seqno_; ec = EPROTO; } if (ec != EINTR) { error_code_ = ec; } - handler_.ist_end(ec); + handler_.ist_end(Result{ec, error_os.str()}); } @@ -776,7 +776,7 @@ if (err != 0) { delete as; - gu_throw_error(err) << "failed to start sender thread"; + gu_throw_system_error(err) << "failed to start sender thread"; } senders_.insert(as); } diff -Nru galera-4-26.4.18/galera/src/ist.hpp galera-4-26.4.20/galera/src/ist.hpp --- galera-4-26.4.18/galera/src/ist.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/ist.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -29,6 +29,15 @@ void register_params(gu::Config& conf); + struct Result + { + int error; + std::string error_str; + Result(int error_arg, const std::string& error_str_arg) + : error{error_arg} + , error_str{error_str_arg} + { } + }; // IST event handler interface class EventHandler { @@ -40,7 +49,7 @@ virtual void ist_cc(const gcs_action&, bool must_apply, bool preload) = 0; // Report IST end - virtual void ist_end(int error) = 0; + virtual void ist_end(const Result&) = 0; protected: virtual ~EventHandler() {} }; diff -Nru galera-4-26.4.18/galera/src/ist_proto.hpp galera-4-26.4.20/galera/src/ist_proto.hpp --- galera-4-26.4.18/galera/src/ist_proto.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/ist_proto.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -724,7 +724,8 @@ } else { - gu_throw_error(-msg.ctrl()) <<"peer reported error"; + gu_throw_error(-msg.ctrl()) + << "peer reported error: " << -msg.ctrl(); } } default: diff -Nru galera-4-26.4.18/galera/src/key_data.hpp galera-4-26.4.20/galera/src/key_data.hpp --- galera-4-26.4.18/galera/src/key_data.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/key_data.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ // -// Copyright (C) 2013-2018 Codership Oy +// Copyright (C) 2013-2024 Codership Oy // #ifndef GALERA_KEY_DATA_HPP @@ -29,7 +29,7 @@ copy (cp) {} - static wsrep_key_type_t const BRANCH_KEY_TYPE = WSREP_KEY_SHARED; + static wsrep_key_type_t const BRANCH_KEY_TYPE = WSREP_KEY_REFERENCE; /* Zero-level key constructor */ explicit @@ -50,7 +50,8 @@ copy (kd.copy) {} - bool shared() const { return type == WSREP_KEY_SHARED; } + bool shared() const { return (type == WSREP_KEY_SHARED); } + bool shared_or_ref() const { return (type <= WSREP_KEY_REFERENCE); } void print(std::ostream& os) const; diff -Nru galera-4-26.4.18/galera/src/key_set.cpp galera-4-26.4.20/galera/src/key_set.cpp --- galera-4-26.4.18/galera/src/key_set.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/key_set.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ // -// Copyright (C) 2013-2021 Codership Oy +// Copyright (C) 2013-2024 Codership Oy // #include "key_set.hpp" @@ -219,7 +219,7 @@ key_prefix_is_stronger_than(int const left, int const right) { - return left > right; // for now key prefix is numerically ordered + return left > right; } KeySetOut::KeyPart::KeyPart (KeyParts& added, @@ -247,51 +247,16 @@ hash_.gather(hd.buf); - /* only leaf part of the key can be not WSREP_KEY_SHARED */ + /* only leaf part of the key can be not of branch type */ bool const leaf (part_num + 1 == kd.parts_num); wsrep_key_type_t const type (leaf ? kd.type : KeyData::BRANCH_KEY_TYPE); int const prefix (KeySet::KeyPart::prefix(type, ws_ver)); +// log_info << "Part " << part_num +1 << '/' << kd.parts_num << ": leaf: " << leaf << ", kd.type: " << kd.type << ", type: " << type << ", prefix: " << prefix; assert (kd.parts_num > part_num); KeySet::KeyPart kp(ts, hd, kd.parts, ver_, prefix, part_num, alignment); -#if 0 /* find() way */ - /* the reason to use find() first, instead of going straight to insert() - * is that we need to insert the part that was persistently stored in the - * key set. At the same time we can't yet store the key part in the key set - * before we can be sure that it is not a duplicate. Sort of 2PC. */ - KeyParts::iterator found(added.find(kp)); - - if (added.end() != found) - { - if (key_prefix_is_stronger_than(prefix, found->prefix())) - { /* need to ditch weaker and add stronger version of the key */ - added.erase(found); - found = added.end(); - } - else if (leaf || key_prefix_is_stronger_than(found->prefix(), prefix)) - { -#ifndef NDEBUG - if (leaf) - log_debug << "KeyPart ctor: full duplicate of " << *found; - else - log_debug << "Duplicate of stronger: " << *found; -#endif - throw DUPLICATE(); - } - } - - if (added.end() == found) /* no such key yet, store and add */ - { - kp.store (store); - std::pair res(added.insert(kp)); - assert (res.second); - found = res.first; - } - - part_ = &(*found); -#else /* insert() way */ std::pair const inserted(added.insert(kp)); if (inserted.second) @@ -334,7 +299,6 @@ } part_ = &(*inserted.first); -#endif /* insert() way */ } void @@ -348,74 +312,88 @@ os << '(' << gu::Hexdump(value_, size_, true) << ')'; } -#define CHECK_PREVIOUS_KEY 1 +/* Uncomment to enable KeySetOut::append() debug logging */ +// #define GALERA_KSO_APPEND_DEBUG 1 +#ifdef GALERA_KSO_APPEND_DEBUG +#define KSO_APPEND_DEBUG(...) log_info << __VA_ARGS__ +#else +#define KSO_APPEND_DEBUG(...) +#endif -size_t -KeySetOut::append (const KeyData& kd) +int KeySetOut::find_common_ancestor_with_previous(const KeyData& kd) const { int i(0); - -// log_info << "Appending key data:" << kd; - -#ifdef CHECK_PREVIOUS_KEY - /* find common ancestor with the previous key */ for (; i < kd.parts_num && size_t(i + 1) < prev_.size() && prev_[i + 1].match(kd.parts[i].ptr, kd.parts[i].len); ++i) { -#if 0 - log_info << "prev[" << (i+1) << "]\n" - << prev_[i+1] - << "\nmatches\n" - << gu::Hexdump(kd.parts[i].ptr, kd.parts[i].len, true); -#endif /* 0 */ + KSO_APPEND_DEBUG("prev[" << (i+1) << "]\n" + << prev_[i+1] + << "\nmatches\n" + << gu::Hexdump(kd.parts[i].ptr, kd.parts[i].len, true)); } -// log_info << "matched " << i << " parts"; + assert(size_t(i) < prev_.size()); + return i; +} - int const kd_leaf_prefix(KeySet::KeyPart::prefix(kd.type, ws_ver_)); +size_t +KeySetOut::append (const KeyData& kd) +{ + int i = find_common_ancestor_with_previous(kd); + KSO_APPEND_DEBUG("Append " << kd); /* if we have a fully matched key OR common ancestor is stronger, return */ if (i > 0) { - assert (size_t(i) < prev_.size()); - + int const kd_leaf_prefix(KeySet::KeyPart::prefix(kd.type, ws_ver_)); + bool const common_ancestor_is_kd_leaf = (kd.parts_num == i); + int const branch_prefix + (KeySet::KeyPart::prefix(KeyData::BRANCH_KEY_TYPE, ws_ver_)); int const exclusive_prefix (KeySet::KeyPart::prefix(WSREP_KEY_EXCLUSIVE, ws_ver_)); + int const common_ancestor_prefix = prev_[i].prefix(); + bool const common_ancestor_is_prev_leaf = (prev_.size() == (i + 1U)); - if (key_prefix_is_stronger_than(prev_[i].prefix(), kd_leaf_prefix) || - prev_[i].prefix() == exclusive_prefix) + KSO_APPEND_DEBUG("Found common ancestor " << prev_[i] << " at position " << i); + + /* The common ancestor is already the strongest possible key. */ + if (common_ancestor_prefix == exclusive_prefix) { -// log_info << "Returning after matching a stronger key:\n"< kd_leaf_prefix + && common_ancestor_prefix > branch_prefix) { - assert(!key_prefix_is_stronger_than(prev_[i].prefix(), - kd_leaf_prefix)); + KSO_APPEND_DEBUG("Common ancestor is previous leaf and stronger"); + return 0; + } - if (prev_[i].prefix() == kd_leaf_prefix) + if (common_ancestor_is_kd_leaf) + { + KSO_APPEND_DEBUG("Common ancestor is kd leaf"); + if (kd_leaf_prefix <= common_ancestor_prefix) { -// log_info << "Returning after matching all " << i << " parts"; + KSO_APPEND_DEBUG("Common ancestor covers kd leaf"); return 0; } - else /* need to add a stronger copy of the leaf */ - --i; + + assert(common_ancestor_prefix <= kd_leaf_prefix); + /* need to add a stronger copy of the leaf */ + --i; } } int const anc(i); + KSO_APPEND_DEBUG("Append key parts after ancestor " << i); const KeyPart* parent(&prev_[anc]); -// log_info << "Common ancestor: " << anc << ' ' << *parent; -#else - KeyPart tmp(prev_[0]); - const KeyPart* const parent(&tmp); -#endif /* CHECK_PREVIOUS_KEY */ - /* create parts that didn't match previous key and add to the set * of previously added keys. */ size_t const old_size (size()); @@ -425,8 +403,6 @@ try { KeyPart kp(added_, *this, parent, kd, i, ws_ver_, alignment()); - -#ifdef CHECK_PREVIOUS_KEY if (size_t(j) < new_.size()) { new_[j] = kp; @@ -436,13 +412,6 @@ new_().push_back (kp); } parent = &new_[j]; -#else - if (kd.copy) kp.acquire(); - if (i + 1 != kd.parts_num) - tmp = kp; // <- updating parent for next iteration -#endif /* CHECK_PREVIOUS_KEY */ - -// log_info << "pushed " << kp; } catch (KeyPart::DUPLICATE& e) { @@ -450,9 +419,6 @@ /* There is a very small probability that child part throws DUPLICATE * even after parent was added as a new key. It does not matter: * a duplicate will be a duplicate in certification as well. */ -#ifndef NDEBUG -// log_debug << "Returning after catching a DUPLICATE. Part: " << i; -#endif /* NDEBUG */ goto out; } } @@ -460,7 +426,6 @@ assert (i == kd.parts_num); assert (anc + j == kd.parts_num); -#ifdef CHECK_PREVIOUS_KEY /* copy new parts to prev_ */ prev_().resize(1 + kd.parts_num); std::copy(new_().begin(), new_().begin() + j, prev_().begin() + anc + 1); @@ -471,28 +436,11 @@ { prev_[k].acquire(); } -#endif /* CHECK_PREVIOUS_KEY */ out: return size() - old_size; } -#if 0 -const KeyIn& -galera::KeySetIn::get_key() const -{ - size_t offset(0); - while (offset < keys_.size()) - { - KeyOS key(version_); - if ((offset = unserialize(&keys_[0], keys_.size(), offset, key)) == 0) - { - gu_throw_fatal << "failed to unserialize key"; - } - s.push_back(key); - } - assert(offset == keys_.size()); -} -#endif +#undef KSO_APPEND_DEBUG } /* namespace galera */ diff -Nru galera-4-26.4.18/galera/src/key_set.hpp galera-4-26.4.20/galera/src/key_set.hpp --- galera-4-26.4.18/galera/src/key_set.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/key_set.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -724,6 +724,7 @@ KeySet::Version version_; int ws_ver_; + int find_common_ancestor_with_previous(const KeyData&) const; static gu::RecordSet::CheckType check_type (KeySet::Version ver) { diff -Nru galera-4-26.4.18/galera/src/mapped_buffer.cpp galera-4-26.4.20/galera/src/mapped_buffer.cpp --- galera-4-26.4.18/galera/src/mapped_buffer.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/mapped_buffer.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -85,21 +85,23 @@ fd_ = mkstemp(&file_[0]); if (fd_ == -1) { - gu_throw_error(errno) << "mkstemp(" << file_ << ") failed"; + gu_throw_system_error(errno) + << "mkstemp(" << file_ << ") failed"; } if (ftruncate(fd_, sz) == -1) { - gu_throw_error(errno) << "ftruncate() failed"; + gu_throw_system_error(errno) << "ftruncate() failed"; } byte_t* tmp(reinterpret_cast( mmap(NULL, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd_, 0))); if (tmp == MAP_FAILED) { + const int error = errno; free(buf_); buf_ = 0; clear(); - gu_throw_error(ENOMEM) << "mmap() failed"; + gu_throw_system_error(error) << "mmap() failed"; } copy(buf_, buf_ + buf_size_, tmp); free(buf_); @@ -109,19 +111,20 @@ { if (munmap(buf_, real_buf_size_) != 0) { - gu_throw_error(errno) << "munmap() failed"; + gu_throw_system_error(errno) << "munmap() failed"; } if (ftruncate(fd_, sz) == -1) { - gu_throw_error(errno) << "fruncate() failed"; + gu_throw_system_error(errno) << "fruncate() failed"; } byte_t* tmp(reinterpret_cast( mmap(NULL, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd_, 0))); if (tmp == MAP_FAILED) { + const int error = errno; buf_ = 0; clear(); - gu_throw_error(ENOMEM) << "mmap() failed"; + gu_throw_system_error(error) << "mmap() failed"; } buf_ = tmp; } @@ -132,7 +135,7 @@ byte_t* tmp(reinterpret_cast(realloc(buf_, sz))); if (tmp == 0) { - gu_throw_error(ENOMEM) << "realloc failed"; + gu_throw_system_error(ENOMEM) << "realloc failed"; } buf_ = tmp; } diff -Nru galera-4-26.4.18/galera/src/replicator_smm.cpp galera-4-26.4.20/galera/src/replicator_smm.cpp --- galera-4-26.4.18/galera/src/replicator_smm.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/replicator_smm.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -5,6 +5,7 @@ #include "galera_common.hpp" #include "replicator_smm.hpp" #include "gcs_action_source.hpp" +#include "gcs_error.hpp" #include "galera_exception.hpp" #include "galera_info.hpp" @@ -309,7 +310,6 @@ delete as_; } - wsrep_status_t galera::ReplicatorSMM::connect(const std::string& cluster_name, const std::string& cluster_url, const std::string& state_donor, @@ -342,14 +342,14 @@ if (ret == WSREP_OK && (err = gcs_.set_initial_position(inpos)) != 0) { - log_error << "gcs init failed:" << strerror(-err); + log_error << "gcs init failed:" << gcs_error_str(-err); ret = WSREP_NODE_FAIL; } if (ret == WSREP_OK && (err = gcs_.connect(cluster_name, cluster_url, bootstrap)) != 0) { - log_error << "gcs connect failed: " << strerror(-err); + log_error << "gcs connect failed: " << gcs_error_str(-err); ret = WSREP_NODE_FAIL; } @@ -1565,8 +1565,8 @@ } catch (gu::Exception& e) { - log_warn << "gcs_caused() returned " << -e.get_errno() - << " (" << strerror(e.get_errno()) << ")"; + log_debug << "gcs_caused() returned " << -e.get_errno() + << " (" << strerror(e.get_errno()) << ")"; return WSREP_TRX_FAIL; } } @@ -1664,7 +1664,7 @@ else if (err < 0) { log_error << "Failed to send NBO-end: " << err << ": " - << ::strerror(-err); + << gcs_error_str(-err); return WSREP_NODE_FAIL; } @@ -1935,7 +1935,8 @@ if (rcode < 0) gu_throw_error(-rcode) - << "Replication of preordered writeset failed."; + << "Replication of preordered writeset failed: " + << gcs_error_str(-rcode); } delete ws; // cleanup regardless of commit flag @@ -2250,7 +2251,7 @@ default: /* general error */ assert(ret < 0); msg << "Failed to vote on request for " << gtid << ": " - << -ret << " (" << ::strerror(-ret) << "). " + << -ret << " (" << gcs_error_str(-ret) << "). " "Assuming inconsistency"; goto fail; } @@ -2574,7 +2575,7 @@ bool galera::ReplicatorSMM::skip_prim_conf_change( const wsrep_view_info_t& view_info, int const proto_ver) { - auto cc_seqno(WSREP_SEQNO_UNDEFINED); + wsrep_seqno_t cc_seqno(WSREP_SEQNO_UNDEFINED); bool keep(false); // keep in cache if (proto_ver >= PROTO_VER_ORDERED_CC) @@ -2586,7 +2587,7 @@ // was not part of IST preload, adjust cert. index // see handle_trx_overlapping_ist() for analogous logic assert(cc_seqno == cert_.position() + 1); - const auto trx_ver + const int trx_ver (std::get<0>(get_trx_protocol_versions(proto_ver))); cert_.adjust_position(view_info, gu::GTID(view_info.state_id.uuid, cc_seqno), @@ -3043,8 +3044,9 @@ if (seqno_j < 0 && S_JOINING == state_()) { // #595, @todo: find a way to re-request state transfer - log_fatal << "Failed to receive state transfer: " << seqno_j - << " (" << strerror (-seqno_j) << "), need to restart."; + log_fatal << "Failed to receive state transfer: " << seqno_j << " (" + << gcs_state_transfer_error_str(-seqno_j) + << "), need to restart."; abort(); } else @@ -3159,7 +3161,7 @@ if (ret) { - gu_throw_error (-ret) << "Node desync failed."; + gu_throw_error(-ret) << gcs_error_str(-ret); } } diff -Nru galera-4-26.4.18/galera/src/replicator_smm.hpp galera-4-26.4.20/galera/src/replicator_smm.hpp --- galera-4-26.4.18/galera/src/replicator_smm.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/replicator_smm.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -180,9 +180,9 @@ // IST Action handler interface void ist_trx(const TrxHandleSlavePtr& ts, bool must_apply, - bool preload); - void ist_cc(const gcs_action&, bool must_apply, bool preload); - void ist_end(int error); + bool preload) override; + void ist_cc(const gcs_action&, bool must_apply, bool preload) override; + void ist_end(const ist::Result&) override; // Cancel local and enter apply monitors for TrxHandle void cancel_monitors_for_local(const TrxHandleSlave& ts) @@ -260,15 +260,15 @@ mutex_(), cond_(), eof_(false), - error_(0), + result_(0, ""), queue_() { } - void reset() { eof_ = false; error_ = 0; } - void eof(int error) + void reset() { eof_ = false; result_ = ist::Result{0, ""}; } + void eof(const ist::Result& result) { gu::Lock lock(mutex_); eof_ = true; - error_ = error; + result_ = result; cond_.broadcast(); } @@ -309,12 +309,14 @@ } else { - if (error_) + if (result_.error) { - int err(error_); - error_ = 0; // Make just one thread to detect the failure + int err(result_.error); + // Make just one thread to detect the failure + result_.error = 0; gu_throw_error(err) - << "IST receiver reported failure"; + << "IST receiver reported failure: '" + << result_.error_str << "' (" << err << ")"; } } @@ -325,7 +327,7 @@ gu::Mutex mutex_; gu::Cond cond_; bool eof_; - int error_; + ist::Result result_; std::queue queue_; }; @@ -870,7 +872,7 @@ ssize_t sst_req_len); /* resume reception of GCS events */ - void resume_recv() { gcs_.resume_recv(); ist_end(0); } + void resume_recv() { gcs_.resume_recv(); ist_end(ist::Result{0, ""}); } /* These methods facilitate closing procedure. * They must be called under closing_mutex_ lock */ diff -Nru galera-4-26.4.18/galera/src/replicator_str.cpp galera-4-26.4.20/galera/src/replicator_str.cpp --- galera-4-26.4.18/galera/src/replicator_str.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/replicator_str.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -4,6 +4,7 @@ #include "replicator_smm.hpp" #include "galera_info.hpp" +#include "gcs_error.hpp" #include #include @@ -805,12 +806,12 @@ if (!retry_str(ret)) { log_error << "Requesting state transfer failed: " - << ret << "(" << strerror(-ret) << ")"; + << gcs_state_transfer_error_str(-ret); } else if (1 == tries) { log_info << "Requesting state transfer failed: " - << ret << "(" << strerror(-ret) << "). " + << gcs_state_transfer_error_str(-ret) << ". " << "Will keep retrying every " << sst_retry_sec_ << " second(s)"; } @@ -862,7 +863,8 @@ if (!closing_ && state_() > S_CLOSED) { log_fatal << "State transfer request failed unrecoverably: " - << -ret << " (" << strerror(-ret) << "). Most likely " + << gcs_state_transfer_error_str(-ret) + << ". Most likely " << "it is due to inability to communicate with the " << "cluster primary component. Restart required."; abort(); @@ -1400,9 +1402,9 @@ } } -void ReplicatorSMM::ist_end(int error) +void ReplicatorSMM::ist_end(const ist::Result& result) { - ist_event_queue_.eof(error); + ist_event_queue_.eof(result); } void galera::ReplicatorSMM::process_ist_conf_change(const gcs_act_cchange& conf) diff -Nru galera-4-26.4.18/galera/src/saved_state.cpp galera-4-26.4.20/galera/src/saved_state.cpp --- galera-4-26.4.18/galera/src/saved_state.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/saved_state.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -48,7 +48,7 @@ if (!fs_) { - gu_throw_error(errno) + gu_throw_system_error(errno) << "Could not open state file for writing: '" << file << "'. Check permissions and/or disk space."; } diff -Nru galera-4-26.4.18/galera/src/trx_handle.hpp galera-4-26.4.20/galera/src/trx_handle.hpp --- galera-4-26.4.18/galera/src/trx_handle.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/trx_handle.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ // -// Copyright (C) 2010-2018 Codership Oy +// Copyright (C) 2010-2024 Codership Oy // @@ -859,19 +859,6 @@ void append_key(const KeyData& key) { - // Current limitations with certification on trx versions 3 to 5 - // impose the the following restrictions on keys - - // The shared key behavior for TOI operations is completely - // untested, so don't allow it (and it probably does not even - // make any sense) - assert(is_toi() == false - || key.shared() == false - || (key.parts_num == 1 - && key.parts->len == 1 - /* this could be a server-level key */) - ); - /*! protection against protocol change during trx lifetime */ if (key.proto_ver != version()) { diff -Nru galera-4-26.4.18/galera/src/wsrep_provider.cpp galera-4-26.4.20/galera/src/wsrep_provider.cpp --- galera-4-26.4.18/galera/src/wsrep_provider.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/src/wsrep_provider.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -16,6 +16,7 @@ #include "wsrep_params.hpp" #include "gu_event_service.hpp" #include "wsrep_config_service.h" +#include "wsrep_node_isolation.h" #include @@ -1476,7 +1477,7 @@ } catch (gu::Exception& e) { - log_error << e.what(); + log_warn << "Node pause failed: " << e.what(); return -e.get_errno(); } } @@ -1497,7 +1498,7 @@ } catch (gu::Exception& e) { - log_error << e.what(); + log_error << "Node resume failed: " << e.what(); return WSREP_NODE_FAIL; } } @@ -1518,7 +1519,7 @@ } catch (gu::Exception& e) { - log_error << e.what(); + log_warn << "Node desync failed: " << e.what(); return WSREP_TRX_FAIL; } } @@ -1539,7 +1540,7 @@ } catch (gu::Exception& e) { - log_error << e.what(); + log_error << "Node resync failed: " << e.what(); return WSREP_NODE_FAIL; } } @@ -1788,3 +1789,20 @@ { gu::Config::enable_deprecation_check(); } + +/* + * This function may be called from signal handler, so make sure that + * only 'safe' system calls and library functions are used. See + * https://pubs.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html + */ +extern "C" enum wsrep_node_isolation_result +wsrep_node_isolation_mode_set_v1(enum wsrep_node_isolation_mode mode) +{ + if (mode < WSREP_NODE_ISOLATION_NOT_ISOLATED + || mode > WSREP_NODE_ISOLATION_FORCE_DISCONNECT) + { + return WSREP_NODE_ISOLATION_INVALID_VALUE; + } + gu::gu_asio_node_isolation_mode = mode; + return WSREP_NODE_ISOLATION_SUCCESS; +} diff -Nru galera-4-26.4.18/galera/tests/certification_check.cpp galera-4-26.4.20/galera/tests/certification_check.cpp --- galera-4-26.4.18/galera/tests/certification_check.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/tests/certification_check.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ // -// Copyright (C) 2015-2021 Codership Oy +// Copyright (C) 2015-2024 Codership Oy // #include "replicator_smm.hpp" // ReplicatorSMM::InitConfig @@ -9,6 +9,7 @@ #include "GCache.hpp" #include "gu_config.hpp" #include "gu_inttypes.hpp" +#include "test_key.hpp" #include @@ -92,6 +93,13 @@ for (size_t i(0); i < nws; ++i) { + log_info << "%%%%%%%% Processing WS: " << i << " ver: " << version + << " l: " << wsi[i].local_seqno + << " g: " << wsi[i].global_seqno + << " s: " << wsi[i].last_seen_seqno + << " leaf: " << (wsi[i].shared ? + WSREP_KEY_REFERENCE : WSREP_KEY_EXCLUSIVE) + << " base: " << wsi[i].zero_level; galera::TrxHandleMasterPtr trx(galera::TrxHandleMaster::New( mp, trx_params, @@ -104,19 +112,19 @@ galera::KeyData(version, wsi[i].key, wsi[i].iov_len, - (wsi[i].shared ? WSREP_KEY_SHARED : + (wsi[i].shared ? galera::KeyData::BRANCH_KEY_TYPE : WSREP_KEY_EXCLUSIVE), true)); if (version >= 6) // version from which zero-level keys were introduced { - if (WSREP_KEY_SHARED != wsi[i].zero_level) + if (galera::KeyData::BRANCH_KEY_TYPE != wsi[i].zero_level) { trx->append_key(galera::KeyData(version, wsi[i].zero_level)); } - // this is always added last in ReplicatorSMM::replicate() - trx->append_key(galera::KeyData(version, WSREP_KEY_SHARED)); + // this is always called last in ReplicatorSMM::replicate() + trx->append_key(galera::KeyData(version)); } if (wsi[i].data_len) @@ -148,12 +156,13 @@ galera::Certification::TestResult result(cert.append_trx(ts)); ck_assert_msg(result == wsi[i].result, - "g: %" PRId64 " res: %d exp: %d", - ts->global_seqno(), result, wsi[i].result); + "g: %" PRId64 " res: %d exp: %d, version: %d", + ts->global_seqno(), result, wsi[i].result, version); ck_assert_msg(ts->depends_seqno() == wsi[i].expected_depends_seqno, - "wsi: %zu g: %" PRId64 " ld: %" PRId64 " eld: %" PRId64, + "wsi: %zu g: %" PRId64 " ld: %" PRId64 " eld: %" PRId64 + ", version: %d", i, ts->global_seqno(), ts->depends_seqno(), - wsi[i].expected_depends_seqno); + wsi[i].expected_depends_seqno, version); cert.set_trx_committed(*ts); if (ts->nbo_end() && ts->ends_nbo() != WSREP_SEQNO_UNDEFINED) @@ -163,11 +172,9 @@ } } - -START_TEST(test_certification_trx_v3) +START_TEST(test_certification_trx_v4) { - - const int version(3); + const int version(4); using galera::Certification; using galera::TrxHandle; using galera::void_cast; @@ -206,7 +213,7 @@ // 5: depends on 4 { { {2, } }, 1, 5, { {void_cast("1"), 1}, {void_cast("1"), 1}, {void_cast("1"), 1} }, 3, false, - 5, 5, 0, 4, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, + 5, 5, 4, 4, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, Certification::TEST_OK, {0}, 0}, // 6 - 8: exclusive - shared @@ -272,16 +279,16 @@ } END_TEST -START_TEST(test_certification_trx_different_level_v3) +START_TEST(test_certification_trx_different_level_v4) { - const int version(3); + const int version(4); using galera::Certification; using galera::TrxHandle; using galera::void_cast; // // Test the following cases: - // 1) exclusive (k1, k2, k3) <-> exclusive (k1, k2) -> dependency + // 1) exclusive (k1, k2, k3) <-> exclusive (k1, k2) -> conflict // 2) exclusive (k1, k2) <-> exclusive (k1, k2, k3) -> conflict // WSInfo wsi[] = { @@ -295,11 +302,11 @@ { {void_cast("1"), 1}, {void_cast("1"), 1} }, 2, false, 2, 2, 0, 1, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, - Certification::TEST_OK, {0}, 0}, + Certification::TEST_FAILED, {0}, 0}, // 2) { { {2, } }, 2, 2, { {void_cast("1"), 1}, {void_cast("1"), 1} }, 2, false, - 3, 3, 2, 2, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, + 3, 3, 2, 1, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, Certification::TEST_OK, {0}, 0}, { { {1, } }, 1, 1, @@ -470,7 +477,7 @@ START_TEST(test_certification_commit_fragment) { - const int version(3); + const int version(4); using galera::Certification; using galera::TrxHandle; using galera::void_cast; @@ -528,13 +535,12 @@ using galera::TrxHandle; using galera::void_cast; - // Interaction of a zero-level non-SHARED key with "regular" transactions - // "Regular" transaction has a zero-level key SHARED, so regarless of TOI or - // non-TOI, shared or exclusive, it shall interact as a SHARED key trx: + // Interaction of a zero-level non-REFERENCE key with "regular" transactions + // "Regular" transaction has a zero-level key, so regarless of TOI or + // non-TOI, shared or exclusive, it shall interact as a REFERENCE key trx: // conflict: - // * SHARED,EXCLUSIVE - EXCLUSIVE depends on SHARED - // * EXCLUSIVE,SHARED - SHARED conflicts with EXCLUSIVE (except for TOI - // which depends) + // * REFERENCE,EXCLUSIVE - EXCLUSIVE conflicts with REFERENCE + // * EXCLUSIVE,REFERENCE - REFERENCE conflicts with EXCLUSIVE WSInfo wsi[] = { // 1: no dependencies { { {1, } }, 1, 1, @@ -542,13 +548,13 @@ 1, 1, 0, 0, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, Certification::TEST_OK, {0}, 0}, - // 2: exclusive zero-level depends on 1 + // 2: exclusive zero-level same source depends on 1 { { {1, } }, 1, 2, {}, 0, true, 2, 2, 0, 1, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, WSREP_KEY_EXCLUSIVE, - Certification::TEST_OK , {0}, 0}, - // 3: shared last seen 1 - conflict with 2 + Certification::TEST_OK, {0}, 0}, + // 3: default zero-level last seen 1 - conflict with 2 { { {2, } }, 1, 3, { {void_cast("1"), 1}, {void_cast("1"), 1}, {void_cast("1"), 1} }, 3, true, 3, 3, 1, 2, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, @@ -561,33 +567,33 @@ galera::KeyData::BRANCH_KEY_TYPE, Certification::TEST_OK, {0}, 0}, // 5: exclusive depends on 4, conflicts with 2 - { { {2, } }, 1, 5, + { { {1, } }, 1, 5, { {void_cast("1"), 1}, {void_cast("1"), 1}, {void_cast("1"), 1} }, 3, false, 5, 5, 0, 4, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, Certification::TEST_FAILED, {0}, 0}, - // 6: shared depends but does not conflict with 2 because from the same source + // 6: reference depends but does not conflict with 2 because same source { { {1, } }, 1, 6, { {void_cast("1"), 1}, {void_cast("1"), 1}, {void_cast("1"), 1} }, 3, true, 6, 6, 1, 2, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, Certification::TEST_OK, {0}, 0}, - // 7: exclusive, saw 2, depends on 6 + // 7: exclusive, saw 2, conflicts with 6 { { {2, } }, 1, 7, { {void_cast("1"), 1}, {void_cast("1"), 1}, {void_cast("1"), 1} }, 3, false, 7, 7, 2, 6, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, - Certification::TEST_OK, {0}, 0}, - // 8: exclusive zero-level depends on exclusive 7 + Certification::TEST_FAILED, {0}, 0}, + // 8: exclusive zero-level depends on 6 because same source { { {1, } }, 1, 8, {}, 0, true, - 8, 8, 4, 7, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, + 8, 8, 4, 6, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, WSREP_KEY_EXCLUSIVE, Certification::TEST_OK, {0}, 0}, // 9: exclusive zero-level conflicts with exclusive zero-level 8 { { {2, } }, 1, 9, {}, 0, true, - 9, 9, 0, 8, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, + 9, 9, 6, 8, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, WSREP_KEY_EXCLUSIVE, Certification::TEST_FAILED, {0}, 0}, // TOI 1 depends on zero-level 8 @@ -604,14 +610,13 @@ TrxHandle::F_ISOLATION | TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, galera::KeyData::BRANCH_KEY_TYPE, Certification::TEST_OK, {0}, 0}, - // zero-level 12 from the different source only depends on TOI 2 - // becaused it is EXCLUSIVE over SHARED + // zero-level 12 from the different source conflicts with TOI 2 { { {2, } }, 3, 3, {}, 0, true, 12, 12, 10, 11, TrxHandle::F_BEGIN | TrxHandle::F_COMMIT, WSREP_KEY_EXCLUSIVE, - Certification::TEST_OK, {0}, 0}, + Certification::TEST_FAILED, {0}, 0}, }; size_t nws(sizeof(wsi)/sizeof(wsi[0])); @@ -621,25 +626,878 @@ } END_TEST +using CertResult = galera::Certification::TestResult; +struct CertFixture +{ + gu::Config conf{}; + struct InitConf + { + galera::ReplicatorSMM::InitConfig init; + InitConf(gu::Config& conf) : init(conf, NULL, NULL) + { + conf.set("gcache.name", "cert_fixture.cache"); + conf.set("gcache.size", "1M"); + } + } init_conf{conf}; + + galera::TrxHandleMaster::Pool mp{ sizeof(galera::TrxHandleMaster) + + sizeof(galera::WriteSetOut), + 16, "certification_mp" }; + galera::TrxHandleSlave::Pool sp{ sizeof(galera::TrxHandleSlave), 16, + "certification_sp" }; + + galera::ProgressCallback gcache_pcb{WSREP_MEMBER_UNDEFINED, + WSREP_MEMBER_UNDEFINED}; + gcache::GCache gcache{&gcache_pcb, conf, "."}; + galera::Certification cert{conf, 0}; + int version = galera::WriteSetNG::MAX_VERSION; + CertFixture() { + cert.assign_initial_position(gu::GTID(), version); + } + + wsrep_uuid_t node1{{1, }}; + wsrep_uuid_t node2{{2, }}; + + wsrep_conn_id_t conn1{1}; + wsrep_conn_id_t conn2{2}; + + wsrep_trx_id_t cur_trx_id{0}; + wsrep_seqno_t cur_seqno{0}; + + struct CfCertResult { + CertResult result; + galera::TrxHandleSlavePtr ts; + }; + + CfCertResult append(const wsrep_uuid_t& node, wsrep_conn_id_t conn, + wsrep_seqno_t last_seen, + const std::vector& key, + wsrep_key_type_t type, int flags, + const gu::byte_t* data_buf, size_t data_buf_len) + { + galera::TrxHandleMasterPtr txm{ galera::TrxHandleMaster::New( + mp, + galera::TrxHandleMaster::Params{ + "", version, + galera::KeySet::MAX_VERSION }, + node, conn, cur_trx_id), + galera::TrxHandleMasterDeleter{} }; + txm->set_flags(flags); + TestKey tkey{ txm->version(), type, key }; + txm->append_key(tkey()); + if (data_buf) + { + txm->append_data(data_buf, data_buf_len, WSREP_DATA_ORDERED, false); + } + galera::WriteSetNG::GatherVector out; + size_t size = txm->write_set_out().gather( + txm->source_id(), txm->conn_id(), txm->trx_id(), out); + txm->finalize(last_seen); + gu::byte_t* buf = static_cast(gcache.malloc(size)); + ck_assert(out.serialize(buf, size) == size); + ++cur_seqno; + gcs_action act = { cur_seqno, cur_seqno, buf, + static_cast(size), GCS_ACT_WRITESET }; + galera::TrxHandleSlavePtr ts(galera::TrxHandleSlave::New(false, sp), + galera::TrxHandleSlaveDeleter{}); + ck_assert(ts->unserialize(act) == size); + auto result = cert.append_trx(ts); + /* Mark committed here to avoid doing it in every test case. If the + * ts is not marked as committed, the certification destructor will + * assert during cleanup. */ + ts->mark_committed(); + return { result, ts }; + } + + CfCertResult append_trx(const wsrep_uuid_t& node, wsrep_conn_id_t conn, + wsrep_seqno_t last_seen, + const std::vector& key, + wsrep_key_type_t type) + { + return append(node, conn, last_seen, key, type, + galera::TrxHandle::F_BEGIN | galera::TrxHandle::F_COMMIT, + nullptr, 0); + } + + CfCertResult append_toi(const wsrep_uuid_t& node, wsrep_conn_id_t conn, + wsrep_seqno_t last_seen, + const std::vector& key, + wsrep_key_type_t type) + { + return append(node, conn, last_seen, key, type, + galera::TrxHandle::F_BEGIN | galera::TrxHandle::F_COMMIT + | galera::TrxHandle::F_ISOLATION, + nullptr, 0); + } + + CfCertResult append_nbo_begin(const wsrep_uuid_t& node, + wsrep_conn_id_t conn, wsrep_seqno_t last_seen, + const std::vector& key, + wsrep_key_type_t type) + { + return append(node, conn, last_seen, key, type, + galera::TrxHandle::F_BEGIN + | galera::TrxHandle::F_ISOLATION, + nullptr, 0); + } + + CfCertResult append_nbo_end(const wsrep_uuid_t& node, wsrep_conn_id_t conn, + wsrep_seqno_t last_seen, + const std::vector& key, + wsrep_key_type_t type, + wsrep_seqno_t begin_seqno) + { + gu::byte_t buf[24]; + galera::NBOKey nbo_key(begin_seqno); + size_t nbo_key_len = nbo_key.serialize(buf, sizeof(buf), 0); + return append(node, conn, last_seen, key, type, + galera::TrxHandle::F_COMMIT + | galera::TrxHandle::F_ISOLATION, + buf, nbo_key_len); + } +}; + +/* This testcase is mainly for checking that the CertFixture works correctly. */ +START_TEST(cert_append_trx) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn2, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert(res.ts->certified()); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); + ck_assert_int_eq(res.ts->global_seqno(), 1); +} +END_TEST + +/* + * Cert against shared + */ + +START_TEST(cert_certify_shared_shared) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_shared_reference) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_shared_update) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_shared_exclusive) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * Cert against reference + */ + +START_TEST(cert_certify_reference_shared) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_reference_reference) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_reference_update) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_reference_exclusive) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * Cert against update + */ + +START_TEST(cert_certify_update_shared) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_update_reference) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_update_update) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_update_exclusive) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * Cert against exclusive + */ + +START_TEST(cert_certify_exclusive_shared) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_exclusive_reference) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_exclusive_update) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_exclusive_exclusive) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * Certify branch against leaf. In these cases the first write set has 2 key + * parts, the second 3 so that the second write set branch key certifies against + * first write set leaf. These are not actually tests for certification, + * but rather for key appending producing proper branch keys. + * Also, in these tests the leaf key for the second transaction does not matter. + */ + +START_TEST(cert_certify_shared_branch) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_reference_branch) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_update_branch) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_exclusive_branch) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* Test certification for branch against other key types. */ + +START_TEST(cert_certify_branch_shared) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b" }, + WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_branch_reference) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b" }, + WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_branch_update) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b" }, + WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_branch_exclusive) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "b" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * TOI shared + */ + +START_TEST(cert_certify_toi_shared_shared) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_toi_shared_reference) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_toi_shared_update) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_shared_exclusive) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * TOI reference + */ + +START_TEST(cert_certify_toi_reference_shared) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_toi_reference_reference) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + +START_TEST(cert_certify_toi_reference_update) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_reference_exclusive) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * TOI update + */ + +START_TEST(cert_certify_toi_update_shared) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_update_reference) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_update_update) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_update_exclusive) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* + * TOI exclusive + */ + +START_TEST(cert_certify_toi_exclusive_shared) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_exclusive_reference) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_REFERENCE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_exclusive_update) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_UPDATE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_toi_exclusive_exclusive) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + + +/* Exclusive - exclusive TOI to demonstrate that TOI never fails + * in certification. */ +START_TEST(cert_certify_exclusive_toi_exclusive) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_toi(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* Exclusive TOI - Exclusive TOI */ +START_TEST(cert_certify_exclusive_toi_exclusive_toi) +{ + CertFixture f; + auto res + = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_toi(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* NBO begin - TOI */ +START_TEST(cert_certify_exclusive_nbo_exclusive_toi) +{ + CertFixture f; + auto res = f.append_nbo_begin(f.node1, f.conn1, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->global_seqno(), 1); + res = f.append_toi(f.node2, f.conn2, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); + res = f.append_nbo_end(f.node1, f.conn1, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE, 1); + res = f.append_toi(f.node2, f.conn2, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 3); +} +END_TEST + +/* TOI - NBO begin */ +START_TEST(cert_certify_exclusive_toi_exclusive_nbo) +{ + CertFixture f; + auto res = f.append_toi(f.node1, f.conn1, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_nbo_begin(f.node2, f.conn2, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->global_seqno(), 2); + ck_assert_int_eq(res.ts->depends_seqno(), 1); + res = f.append_nbo_end(f.node1, f.conn1, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE, 2); + res = f.append_toi(f.node2, f.conn2, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 3); +} +END_TEST + +/* NBO begin - NBO begin*/ +START_TEST(cert_certify_exclusive_nbo_exclusive_nbo) +{ + CertFixture f; + auto res = f.append_nbo_begin(f.node1, f.conn1, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->global_seqno(), 1); + res = f.append_nbo_begin(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + ck_assert_int_eq(res.ts->depends_seqno(), 1); + res = f.append_nbo_end(f.node1, f.conn1, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE, 1); + res = f.append_nbo_begin(f.node2, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 3); +} +END_TEST + +/* Write sets originating from the same node should not conflict even with + * exclusive key. */ +START_TEST(cert_certify_same_node) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node1, f.conn2, 0, { "b", "l" }, + WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* Write set outside certification range must not cause conflict, but dependency. + */ +START_TEST(cert_certify_exclusive_exclusive_outside_cert_range) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 1, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +START_TEST(cert_certify_exclusive_exclusive_shadowed_by_shared) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 1, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); + + res = f.append_trx(f.node2, f.conn2, 0, {"b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_FAILED); + /* Note that even though the dependency should be to shared key, the + * certification checks first for exclusive key and because of conflict, + * the scan stops there and the depends seqno is not updated. This does + * not matter however, as the test result is failed. */ + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* Even though shared-shared match does not cause conflict or dependency, + * having PA_UNSAFE flag in write set must create the dependency. */ +START_TEST(cert_certify_shared_shared_pa_unsafe) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + + res = f.append(f.node2, f.conn2, 0, { "b", "l" }, WSREP_KEY_SHARED, + galera::TrxHandle::F_BEGIN | galera::TrxHandle::F_COMMIT + | galera::TrxHandle::F_PA_UNSAFE, + nullptr, 0); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); +} +END_TEST + +/* PA unsafe must create dependency even if there is no match. */ +START_TEST(cert_certify_no_match_pa_unsafe) +{ + CertFixture f; + auto res = f.append_trx(f.node1, f.conn1, 0, {"b", "m"}, WSREP_KEY_SHARED); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + + res = f.append(f.node2, f.conn2, 0, { "b", "l" }, WSREP_KEY_SHARED, + galera::TrxHandle::F_BEGIN | galera::TrxHandle::F_COMMIT + | galera::TrxHandle::F_PA_UNSAFE, + nullptr, 0); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 1); + +} +END_TEST + +START_TEST(cert_certify_no_match) +{ + CertFixture f; + auto res + = f.append_trx(f.node1, f.conn1, 0, { "b", "m" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + res = f.append_trx(f.node2, f.conn2, 0, { "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(res.result, CertResult::TEST_OK); + ck_assert_int_eq(res.ts->depends_seqno(), 0); +} +END_TEST + + Suite* certification_suite() { Suite* s(suite_create("certification")); TCase* t; - t = tcase_create("certification_trx_v3"); - tcase_add_test(t, test_certification_trx_v3); + t = tcase_create("certification_trx_v4"); + tcase_add_test(t, test_certification_trx_v4); suite_add_tcase(s, t); t = tcase_create("certification_toi_v3"); tcase_add_test(t, test_certification_toi_v3); suite_add_tcase(s, t); - t = tcase_create("certification_trx_different_level_v3"); - tcase_add_test(t, test_certification_trx_different_level_v3); - suite_add_tcase(s, t); - - t = tcase_create("certification_toi_v3"); - tcase_add_test(t, test_certification_toi_v3); + t = tcase_create("certification_trx_different_level_v4"); + tcase_add_test(t, test_certification_trx_different_level_v4); suite_add_tcase(s, t); t = tcase_create("certification_nbo"); @@ -654,5 +1512,61 @@ tcase_add_test(t, test_certification_zero_level); suite_add_tcase(s, t); + t = tcase_create("certification_rules"); + tcase_add_test(t, cert_append_trx); + tcase_add_test(t, cert_certify_shared_shared); + tcase_add_test(t, cert_certify_shared_reference); + tcase_add_test(t, cert_certify_shared_update); + tcase_add_test(t, cert_certify_shared_exclusive); + tcase_add_test(t, cert_certify_reference_shared); + tcase_add_test(t, cert_certify_reference_reference); + tcase_add_test(t, cert_certify_reference_update); + tcase_add_test(t, cert_certify_reference_exclusive); + tcase_add_test(t, cert_certify_update_shared); + tcase_add_test(t, cert_certify_update_reference); + tcase_add_test(t, cert_certify_update_update); + tcase_add_test(t, cert_certify_update_exclusive); + tcase_add_test(t, cert_certify_exclusive_shared); + tcase_add_test(t, cert_certify_exclusive_reference); + tcase_add_test(t, cert_certify_exclusive_update); + tcase_add_test(t, cert_certify_exclusive_exclusive); + tcase_add_test(t, cert_certify_shared_branch); + tcase_add_test(t, cert_certify_reference_branch); + tcase_add_test(t, cert_certify_update_branch); + tcase_add_test(t, cert_certify_exclusive_branch); + tcase_add_test(t, cert_certify_branch_shared); + tcase_add_test(t, cert_certify_branch_reference); + tcase_add_test(t, cert_certify_branch_update); + tcase_add_test(t, cert_certify_branch_exclusive); + tcase_add_test(t, cert_certify_toi_shared_shared); + tcase_add_test(t, cert_certify_toi_shared_reference); + tcase_add_test(t, cert_certify_toi_shared_update); + tcase_add_test(t, cert_certify_toi_shared_exclusive); + tcase_add_test(t, cert_certify_toi_reference_shared); + tcase_add_test(t, cert_certify_toi_reference_reference); + tcase_add_test(t, cert_certify_toi_reference_update); + tcase_add_test(t, cert_certify_toi_reference_exclusive); + tcase_add_test(t, cert_certify_toi_update_shared); + tcase_add_test(t, cert_certify_toi_update_reference); + tcase_add_test(t, cert_certify_toi_update_update); + tcase_add_test(t, cert_certify_toi_update_exclusive); + tcase_add_test(t, cert_certify_toi_exclusive_shared); + tcase_add_test(t, cert_certify_toi_exclusive_reference); + tcase_add_test(t, cert_certify_toi_exclusive_update); + tcase_add_test(t, cert_certify_toi_exclusive_exclusive); + tcase_add_test(t, cert_certify_exclusive_toi_exclusive); + tcase_add_test(t, cert_certify_exclusive_toi_exclusive_toi); + tcase_add_test(t, cert_certify_exclusive_nbo_exclusive_toi); + tcase_add_test(t, cert_certify_exclusive_toi_exclusive_nbo); + tcase_add_test(t, cert_certify_exclusive_nbo_exclusive_nbo); + tcase_add_test(t, cert_certify_same_node); + tcase_add_test(t, cert_certify_exclusive_exclusive_outside_cert_range); + tcase_add_test(t, cert_certify_exclusive_exclusive_shadowed_by_shared); + tcase_add_test(t, cert_certify_shared_shared_pa_unsafe); + tcase_add_test(t, cert_certify_no_match_pa_unsafe); + tcase_add_test(t, cert_certify_no_match); + + suite_add_tcase(s, t); + return s; } diff -Nru galera-4-26.4.18/galera/tests/ist_check.cpp galera-4-26.4.20/galera/tests/ist_check.cpp --- galera-4-26.4.18/galera/tests/ist_check.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/tests/ist_check.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -313,7 +313,8 @@ ~ISTHandler() {} - void ist_trx(const TrxHandleSlavePtr& ts, bool must_apply, bool preload) + void ist_trx(const TrxHandleSlavePtr& ts, bool must_apply, + bool preload) override { assert(ts != 0); ts->verify_checksum(); @@ -339,7 +340,8 @@ seqno_ = ts->global_seqno(); } - void ist_cc(const gcs_action& act, bool must_apply, bool preload) + void ist_cc(const gcs_action& act, bool must_apply, + bool preload) override { gcs_act_cchange const cc(act.buf, act.size); assert(act.seqno_g == cc.seqno); @@ -355,11 +357,11 @@ } } - void ist_end(int error) + void ist_end(const ist::Result& result) override { - log_info << "IST ended with status: " << error; + log_info << "IST ended with status: " << result.error_str; gu::Lock lock(mutex_); - error_ = error; + error_ = result.error; eof_ = true; cond_.signal(); } diff -Nru galera-4-26.4.18/galera/tests/key_set_check.cpp galera-4-26.4.20/galera/tests/key_set_check.cpp --- galera-4-26.4.18/galera/tests/key_set_check.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/tests/key_set_check.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,4 +1,4 @@ -/* copyright (C) 2013-2020 Codership Oy +/* copyright (C) 2013-2024 Codership Oy * * $Id$ */ @@ -7,6 +7,7 @@ #include "test_key.hpp" #include "../src/key_set.hpp" +#include "../src/write_set_ng.hpp" #include "gu_logger.hpp" #include "gu_hexdump.hpp" @@ -73,8 +74,9 @@ TestKey tk1(tk_ver, WSREP_KEY_SHARED, true, "a0", "a1", "a2"); mark_point(); kso.append(tk1()); - ck_assert_msg(kso.count() == 3, "key count: expected 3, got %d", - kso.count()); + int expected_count(3); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); total_size += base_size + 2 + 2*4; total_size = GU_ALIGN(total_size, alignment); @@ -85,19 +87,24 @@ TestKey tk2(tk_ver, WSREP_KEY_EXCLUSIVE, false, "a0", "a1", "b2"); kso.append(tk2()); - ck_assert_msg(kso.count() == 4, "key count: expected 4, got %d", - kso.count()); + expected_count += (ws_ver > 3); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); - total_size += base_size + 2 + 3*4; - total_size = GU_ALIGN(total_size, alignment); + if (expected_count == 4) + { + total_size += base_size + 2 + 3*4; + total_size = GU_ALIGN(total_size, alignment); + } ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); /* this should update a sronger version of "a2" */ - TestKey tk2_(tk_ver, WSREP_KEY_REFERENCE, false, "a0", "a1", "a2"); + TestKey tk2_(tk_ver, WSREP_KEY_UPDATE, false, "a0", "a1", "a2"); kso.append(tk2_()); - ck_assert_msg(kso.count() == 5, "key count: expected 5, got %d", - kso.count()); + expected_count += 1; + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); total_size += base_size + 2 + 3*4; total_size = GU_ALIGN(total_size, alignment); @@ -108,12 +115,16 @@ TestKey tk3(tk_ver, WSREP_KEY_EXCLUSIVE, true, "a0", "a1"); log_info << "######## Appending exclusive duplicate tk3: begin"; kso.append(tk3()); + expected_count += (ws_ver <= 3 ? 0 : 1); log_info << "######## Appending exclusive duplicate tk3: end"; - ck_assert_msg(kso.count() == 6, "key count: expected 6, got %d", - kso.count()); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); - total_size += base_size + 2 + 2*4; - total_size = GU_ALIGN(total_size, alignment); + if (ws_ver > 3) + { + total_size += base_size + 2 + 2*4; + total_size = GU_ALIGN(total_size, alignment); + } ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -122,8 +133,8 @@ log_info << "######## Appending exclusive duplicate tk4: begin"; kso.append(tk4()); log_info << "######## Appending exclusive duplicate tk4: end"; - ck_assert_msg(kso.count() == 6, "key count: expected 6, got %d", - kso.count()); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -131,8 +142,8 @@ /* adding shared key should have no effect */ TestKey tk5(tk_ver, WSREP_KEY_SHARED, true, "a0", "a1"); kso.append(tk5()); - ck_assert_msg(kso.count() == 6, "key count: expected 6, got %d", - kso.count()); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -140,8 +151,8 @@ /* adding REFERENCE key should have no effect */ TestKey tk5_1(tk_ver, WSREP_KEY_REFERENCE, true, "a0", "a1"); kso.append(tk5_1()); - ck_assert_msg(kso.count() == 6, "key count: expected 6, got %d", - kso.count()); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -149,8 +160,8 @@ /* adding UPDATE key should have no effect */ TestKey tk5_2(tk_ver, WSREP_KEY_UPDATE, true, "a0", "a1"); kso.append(tk5_2()); - ck_assert_msg(kso.count() == 6, "key count: expected 6, got %d", - kso.count()); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -158,8 +169,8 @@ /* tk5 should not make any changes */ TestKey tk6(tk_ver, WSREP_KEY_EXCLUSIVE, false, "a0", "a1", "c2"); kso.append(tk6()); - ck_assert_msg(kso.count() == 6, "key count: expected 6, got %d", - kso.count()); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -167,8 +178,9 @@ /* a0:b1:... should still be possible, should add 2 keys: b1 and c2 */ TestKey tk7(tk_ver, WSREP_KEY_REFERENCE, true, "a0", "b1", "c2"); kso.append(tk7()); - ck_assert_msg(kso.count() == 8, "key count: expected 8, got %d", - kso.count()); + expected_count += 2; + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); total_size += base_size + 2 + 2*4; total_size = GU_ALIGN(total_size, alignment); @@ -181,15 +193,18 @@ * (should be no collision on b2) */ TestKey tk8(tk_ver, WSREP_KEY_REFERENCE, false, "a0", "b1", "b2"); kso.append(tk8()); - ck_assert_msg(kso.count() == 9, "key count: expected 9, got %d", - kso.count()); + expected_count += (ws_ver > 3); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); - total_size += base_size + 2 + 3*4; - total_size = GU_ALIGN(total_size, alignment); + if (ws_ver > 3) + { + total_size += base_size + 2 + 3*4; + total_size = GU_ALIGN(total_size, alignment); + } ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); - int expected_count(kso.count()); TestKey tk8_1(tk_ver, WSREP_KEY_UPDATE, false, "a0", "b1", "b2"); kso.append(tk8_1()); if (3 == ws_ver || 4 == ws_ver) @@ -208,8 +223,7 @@ } else abort(); - ck_assert_msg(kso.count() == expected_count, - "key count: expected %d, got %d", + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", expected_count, kso.count()); ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -242,8 +256,7 @@ kso.append(tk8_3()); /* UPDATE key is weaker than EXCLUSIVE, should be ignored */ - ck_assert_msg(kso.count() == expected_count, - "key count: expected %d, got %d", + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", expected_count, kso.count()); ck_assert_msg(total_size == kso.size(), "Size: %zu, expected: %zu", kso.size(), total_size); @@ -270,6 +283,31 @@ log_info << "End size: " << kso.size(); + // Verify that SHARED keys are added as a first leaf bunt not over REFERENCE + TestKey tk10_ref1(tk_ver, WSREP_KEY_REFERENCE, true, "s0"); + kso.append(tk10_ref1()); + expected_count += 1; + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); + // Should add SHARED even though s1 weaker than s0 (for ver > 3) + TestKey tk10_sh(tk_ver, WSREP_KEY_SHARED, true, "s0", "s1"); + kso.append(tk10_sh()); + expected_count += (ws_ver > 3); // at ver<=3 REF is considered EXC + assert(kso.count() == expected_count); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); + TestKey tk10_ref2(tk_ver, WSREP_KEY_REFERENCE, true, "s0", "s1"); + kso.append(tk10_ref2()); + expected_count += (ws_ver > 3); + assert(kso.count() == expected_count); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); + // Try same SHARED once again, should not add anything + kso.append(tk10_sh()); + ck_assert_msg(kso.count() == expected_count, "key count: expected %d, got %d", + expected_count, kso.count()); + + KeySetOut::GatherVector out; out->reserve(kso.page_count()); size_t const out_size(kso.gather(out)); @@ -304,14 +342,14 @@ ck_abort_msg("%s", e.what()); } - int shared(0); // to stiffle clang complaints about unused variables + int branch(0); // to stiffle clang complaints about unused variables - int const P_SHARED(KeySet::KeyPart::prefix(WSREP_KEY_SHARED, ws_ver)); + int const P_BRANCH(KeySet::KeyPart::prefix(KeyData::BRANCH_KEY_TYPE,ws_ver)); for (int i(0); i < ksi.count(); ++i) { KeySet::KeyPart kp(ksi.next()); - shared += (kp.prefix() == P_SHARED); + branch += (kp.prefix() == P_BRANCH); } KeySetIn ksi_empty; @@ -342,7 +380,7 @@ for (int i(0); i < ksi_empty.count(); ++i) { KeySet::KeyPart kp(ksi_empty.next()); - shared += (kp.prefix() == P_SHARED); + branch += (kp.prefix() == P_BRANCH); } ksi_empty.rewind(); @@ -350,10 +388,10 @@ for (int i(0); i < ksi_empty.count(); ++i) { KeySet::KeyPart kp(ksi_empty.next()); - shared += (kp.prefix() == P_SHARED); + branch += (kp.prefix() == P_BRANCH); } - ck_assert(0 != shared); + ck_assert(0 != branch); } #ifndef GALERA_ONLY_ALIGNED @@ -382,6 +420,402 @@ } END_TEST +struct KsoFixture +{ + union Res + { + gu::byte_t buf[1024]; + gu_word_t align; + }; + Res res{}; + TestBaseName basename{ "ksof" }; + galera::KeySetOut kso{ res.buf, + sizeof(res.buf), + basename, + galera::KeySet::FLAT8A, + gu::RecordSet::VER2, + galera::WriteSetNG::MAX_VERSION }; + void append(const std::vector key, wsrep_key_type_t type) + { + + TestKey k{ galera::KeySet::FLAT8A, type, key }; + kso.append(k()); + } +}; + +/* + * Shared leaf + */ + +START_TEST(kso_append_shared_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_shared_over_shared_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_SHARED); + f.append({ "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_reference_over_shared_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_SHARED); + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_update_over_shared_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_SHARED); + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_exclusive_over_shared_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_SHARED); + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_shared_branch_over_shared_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_SHARED); + f.append({"b", "b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_reference_branch_over_shared_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_SHARED); + f.append({"b", "b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_update_branch_over_shared_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_SHARED); + f.append({"b", "b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_exclusive_branch_over_shared_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_SHARED); + f.append({"b", "b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_shared_leaf_over_branch) +{ + KsoFixture f; + f.append({"b", "l"}, WSREP_KEY_SHARED); + f.append({"b"}, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +/* + * Reference leaf + */ + +START_TEST(kso_append_reference_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_shared_over_reference_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + f.append({ "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_reference_over_reference_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_update_over_reference_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_exclusive_over_reference_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_shared_branch_over_reference_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_REFERENCE); + f.append({"b", "b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_reference_branch_over_reference_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_REFERENCE); + f.append({"b", "b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_update_branch_over_reference_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_REFERENCE); + f.append({"b", "b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_exclusive_branch_over_reference_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_REFERENCE); + f.append({"b", "b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_reference_leaf_over_branch) +{ + KsoFixture f; + f.append({"b", "l"}, WSREP_KEY_SHARED); + f.append({"b"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +/* + * Update leaf + */ + +START_TEST(kso_append_update_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_shared_over_update_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + f.append({ "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_reference_over_update_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_update_over_update_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_exclusive_over_update_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_shared_branch_over_update_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_UPDATE); + f.append({"b", "b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_reference_branch_over_update_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_UPDATE); + f.append({"b", "b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_update_branch_over_update_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_UPDATE); + f.append({"b", "b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_exclusive_branch_over_update_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_UPDATE); + f.append({"b", "b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + +START_TEST(kso_append_update_leaf_over_branch) +{ + KsoFixture f; + f.append({"b", "l"}, WSREP_KEY_SHARED); + f.append({"b"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + + +/* + * Exclusive leaf + */ + +START_TEST(kso_append_exclusive_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_shared_over_exclusive_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + f.append({ "b", "l" }, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_reference_over_exclusive_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + f.append({ "b", "l" }, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_update_over_exclusive_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + f.append({ "b", "l" }, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_exclusive_over_exclusive_leaf) +{ + KsoFixture f; + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + f.append({ "b", "l" }, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_shared_branch_over_exclusive_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_EXCLUSIVE); + f.append({"b", "b", "l"}, WSREP_KEY_SHARED); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_reference_branch_over_exclusive_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_EXCLUSIVE); + f.append({"b", "b", "l"}, WSREP_KEY_REFERENCE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_update_branch_over_exclusive_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_EXCLUSIVE); + f.append({"b", "b", "l"}, WSREP_KEY_UPDATE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_exclusive_branch_over_exclusive_leaf) +{ + KsoFixture f; + f.append({"b", "b"}, WSREP_KEY_EXCLUSIVE); + f.append({"b", "b", "l"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 2); +} +END_TEST + +START_TEST(kso_append_exclusive_leaf_over_branch) +{ + KsoFixture f; + f.append({"b", "l"}, WSREP_KEY_SHARED); + f.append({"b"}, WSREP_KEY_EXCLUSIVE); + ck_assert_int_eq(f.kso.count(), 3); +} +END_TEST + Suite* key_set_suite () { TCase* t = tcase_create ("KeySet"); @@ -393,6 +827,51 @@ tcase_add_test (t, ver2_5); tcase_set_timeout(t, 60); + + tcase_add_test(t, kso_append_shared_leaf); + tcase_add_test(t, kso_append_shared_over_shared_leaf); + tcase_add_test(t, kso_append_reference_over_shared_leaf); + tcase_add_test(t, kso_append_update_over_shared_leaf); + tcase_add_test(t, kso_append_exclusive_over_shared_leaf); + tcase_add_test(t, kso_append_shared_branch_over_shared_leaf); + tcase_add_test(t, kso_append_reference_branch_over_shared_leaf); + tcase_add_test(t, kso_append_update_branch_over_shared_leaf); + tcase_add_test(t, kso_append_exclusive_branch_over_shared_leaf); + tcase_add_test(t, kso_append_shared_leaf_over_branch); + + tcase_add_test(t, kso_append_reference_leaf); + tcase_add_test(t, kso_append_shared_over_reference_leaf); + tcase_add_test(t, kso_append_reference_over_reference_leaf); + tcase_add_test(t, kso_append_update_over_reference_leaf); + tcase_add_test(t, kso_append_exclusive_over_reference_leaf); + tcase_add_test(t, kso_append_shared_branch_over_reference_leaf); + tcase_add_test(t, kso_append_reference_branch_over_reference_leaf); + tcase_add_test(t, kso_append_update_branch_over_reference_leaf); + tcase_add_test(t, kso_append_exclusive_branch_over_reference_leaf); + tcase_add_test(t, kso_append_reference_leaf_over_branch); + + tcase_add_test(t, kso_append_update_leaf); + tcase_add_test(t, kso_append_shared_over_update_leaf); + tcase_add_test(t, kso_append_reference_over_update_leaf); + tcase_add_test(t, kso_append_update_over_update_leaf); + tcase_add_test(t, kso_append_exclusive_over_update_leaf); + tcase_add_test(t, kso_append_shared_branch_over_update_leaf); + tcase_add_test(t, kso_append_reference_branch_over_update_leaf); + tcase_add_test(t, kso_append_update_branch_over_update_leaf); + tcase_add_test(t, kso_append_exclusive_branch_over_update_leaf); + tcase_add_test(t, kso_append_update_leaf_over_branch); + + tcase_add_test(t, kso_append_exclusive_leaf); + tcase_add_test(t, kso_append_shared_over_exclusive_leaf); + tcase_add_test(t, kso_append_reference_over_exclusive_leaf); + tcase_add_test(t, kso_append_update_over_exclusive_leaf); + tcase_add_test(t, kso_append_exclusive_over_exclusive_leaf); + tcase_add_test(t, kso_append_shared_branch_over_exclusive_leaf); + tcase_add_test(t, kso_append_reference_branch_over_exclusive_leaf); + tcase_add_test(t, kso_append_update_branch_over_exclusive_leaf); + tcase_add_test(t, kso_append_exclusive_branch_over_exclusive_leaf); + tcase_add_test(t, kso_append_exclusive_leaf_over_branch); + Suite* s = suite_create ("KeySet"); suite_add_tcase (s, t); diff -Nru galera-4-26.4.18/galera/tests/test_key.hpp galera-4-26.4.20/galera/tests/test_key.hpp --- galera-4-26.4.18/galera/tests/test_key.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/tests/test_key.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2013-2018 Codership Oy +/* Copyright (C) 2013-2024 Codership Oy * * $Id$ */ @@ -19,10 +19,10 @@ { public: - TestKey (int a, int ver, - wsrep_key_type_t type, - std::vector parts, - bool copy = true) + TestKey (int const ver, + wsrep_key_type_t const type, + std::vector const parts, + bool const copy = true) : parts_ (), ver_ (ver), @@ -39,19 +39,19 @@ } } - TestKey (int ver, - wsrep_key_type_t type, - bool copy, - const char* part0, - const char* part1 = 0, - const char* part2 = 0, - const char* part3 = 0, - const char* part4 = 0, - const char* part5 = 0, - const char* part6 = 0, - const char* part7 = 0, - const char* part8 = 0, - const char* part9 = 0 + TestKey (int const ver, + wsrep_key_type_t const type, + bool const copy, + const char* const part0, + const char* const part1 = 0, + const char* const part2 = 0, + const char* const part3 = 0, + const char* const part4 = 0, + const char* const part5 = 0, + const char* const part6 = 0, + const char* const part7 = 0, + const char* const part8 = 0, + const char* const part9 = 0 ) : parts_ (), diff -Nru galera-4-26.4.18/galera/tests/write_set_ng_check.cpp galera-4-26.4.20/galera/tests/write_set_ng_check.cpp --- galera-4-26.4.18/galera/tests/write_set_ng_check.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galera/tests/write_set_ng_check.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2013-2020 Codership Oy +/* Copyright (C) 2013-2024 Codership Oy * * $Id$ */ @@ -34,8 +34,9 @@ ck_assert(wso.is_empty()); // keep WSREP_KEY_SHARED here, see loop below - TestKey tk0(KeySet::MAX_VERSION, WSREP_KEY_SHARED, true, "a0"); + TestKey tk0(KeySet::MAX_VERSION, WSREP_KEY_SHARED, true, "a0", "a1"); wso.append_key(tk0()); + int const expected_count(2); ck_assert(wso.is_empty() == false); uint64_t const data_out_volatile(0xaabbccdd); @@ -77,6 +78,7 @@ gu::Buf const in_buf = { in.data(), static_cast(in.size()) }; + int const P_BRANCH(KeySet::KeyPart::prefix(KeyData::BRANCH_KEY_TYPE, wsv)); int const P_SHARED(KeySet::KeyPart::prefix(WSREP_KEY_SHARED, wsv)); /* read ws buffer and "certify" */ @@ -96,15 +98,18 @@ mark_point(); const KeySetIn& ksi(wsi.keyset()); - ck_assert(ksi.count() == 1); + ck_assert(ksi.count() == expected_count); mark_point(); + int branch(0); int shared(0); for (int i(0); i < ksi.count(); ++i) { KeySet::KeyPart kp(ksi.next()); + branch += (kp.prefix() == P_BRANCH); shared += (kp.prefix() == P_SHARED); } + ck_assert(branch > 0); ck_assert(shared > 0); wsi.verify_checksum(); @@ -127,16 +132,16 @@ mark_point(); const KeySetIn& ksi(wsi.keyset()); - ck_assert(ksi.count() == 1); + ck_assert(ksi.count() == expected_count); mark_point(); - int shared(0); + int branch(0); for (int i(0); i < ksi.count(); ++i) { KeySet::KeyPart kp(ksi.next()); - shared += (kp.prefix() == P_SHARED); + branch += (kp.prefix() == P_BRANCH); } - ck_assert(shared > 0); + ck_assert(branch > 0); wsi.verify_checksum(); diff -Nru galera-4-26.4.18/galerautils/src/gu_asio.cpp galera-4-26.4.20/galerautils/src/gu_asio.cpp --- galera-4-26.4.18/galerautils/src/gu_asio.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_asio.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -245,6 +245,11 @@ return (os << ec.message()); } +gu::AsioErrorCode gu::AsioErrorCode::make_eof() +{ + return {asio::error::misc_errors::eof, gu_asio_misc_category}; +} + bool gu::AsioErrorCode::is_eof() const { return (category_ && @@ -317,7 +322,7 @@ if (ifs.good() == false) { - gu_throw_error(errno) << + gu_throw_system_error(errno) << "could not open password file '" << file << "'"; } @@ -906,3 +911,7 @@ --gu_allowlist_service_usage; if (gu_allowlist_service_usage == 0) gu_allowlist_service = 0; } + +std::atomic gu::gu_asio_node_isolation_mode{ + WSREP_NODE_ISOLATION_NOT_ISOLATED +}; diff -Nru galera-4-26.4.18/galerautils/src/gu_asio.hpp galera-4-26.4.20/galerautils/src/gu_asio.hpp --- galera-4-26.4.18/galerautils/src/gu_asio.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_asio.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ // -// Copyright (C) 2014-2020 Codership Oy +// Copyright (C) 2014-2024 Codership Oy // @@ -15,10 +15,12 @@ #include "gu_signals.hpp" #include "wsrep_allowlist_service.h" +#include "wsrep_node_isolation.h" #include // tcp_info #include +#include #include #include #include @@ -242,6 +244,8 @@ bool operator!() const { return value_ == 0; } + + static AsioErrorCode make_eof(); /** * Return true if the error is end of file. */ @@ -295,7 +299,7 @@ /** * This will be called after asynchronous connection to the * remote endpoint after call to AsioSocket::async_connect() - * completes. + * completes, or after accepted socket becomes ready. * * All internal protocol handshakes (e.g. SSL) will be completed * before this handler is called. @@ -601,6 +605,7 @@ virtual void listen(const gu::URI& uri) = 0; virtual void close() = 0; virtual void async_accept(const std::shared_ptr&, + const std::shared_ptr&, const std::shared_ptr& engine = nullptr) = 0; virtual std::shared_ptr accept() = 0; virtual std::string listen_addr() const = 0; @@ -773,6 +778,10 @@ /* Init/deinit global allowlist service hooks. */ int init_allowlist_service_v1(wsrep_allowlist_service_v1_t*); void deinit_allowlist_service_v1(); + /* Global isolation mode. */ + extern std::atomic + gu_asio_node_isolation_mode; + } #endif // GU_ASIO_HPP diff -Nru galera-4-26.4.18/galerautils/src/gu_asio_datagram.cpp galera-4-26.4.20/galerautils/src/gu_asio_datagram.cpp --- galera-4-26.4.18/galerautils/src/gu_asio_datagram.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_asio_datagram.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -104,7 +104,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "error opening datagram socket" << uri; } } @@ -117,7 +117,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "error opening datagram socket" << uri; } } @@ -185,7 +185,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed to connect UDP socket: " << e.what(); } } @@ -201,7 +201,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed to write UDP socket: " << e.what(); } @@ -221,7 +221,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed to send datagram to " << target_endpoint << ": " << e.what(); } diff -Nru galera-4-26.4.18/galerautils/src/gu_asio_socket_util.hpp galera-4-26.4.20/galerautils/src/gu_asio_socket_util.hpp --- galera-4-26.4.18/galerautils/src/gu_asio_socket_util.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_asio_socket_util.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -38,7 +38,7 @@ long flags(FD_CLOEXEC); if (fcntl(native_socket_handle(socket), F_SETFD, flags) == -1) { - gu_throw_error(errno) << "failed to set FD_CLOEXEC"; + gu_throw_system_error(errno) << "failed to set FD_CLOEXEC"; } } @@ -58,7 +58,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed to set receive buffer size: " << e.what(); } @@ -75,7 +75,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed to get receive buffer size: " << e.what(); } @@ -90,7 +90,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed to set send buffer size: " << e.what(); } @@ -107,7 +107,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed to get send buffer size: " << e.what(); } @@ -137,7 +137,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "Failed bind socket to address: " << e.what(); } @@ -159,8 +159,8 @@ if (getsockopt(native_fd, level, TCP_INFO, &tcpi, &tcpi_len)) { int err(errno); - gu_throw_error(err) << "Failed to read TCP info from socket: " - << strerror(err); + gu_throw_system_error(err) << "Failed to read TCP info from socket: " + << strerror(err); } #endif /* __linux__ || __FreeBSD__ */ return tcpi; diff -Nru galera-4-26.4.18/galerautils/src/gu_asio_stream_react.cpp galera-4-26.4.20/galerautils/src/gu_asio_stream_react.cpp --- galera-4-26.4.18/galerautils/src/gu_asio_stream_react.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_asio_stream_react.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ // -// Copyright (C) 2020 Codership Oy +// Copyright (C) 2020-2024 Codership Oy // #define GU_ASIO_IMPL @@ -21,6 +21,20 @@ #include +static bool is_isolated() +{ + const auto mode + = gu::gu_asio_node_isolation_mode.load(std::memory_order_relaxed); + switch (mode) + { + case WSREP_NODE_ISOLATION_NOT_ISOLATED: return false; + case WSREP_NODE_ISOLATION_ISOLATED: return true; + case WSREP_NODE_ISOLATION_FORCE_DISCONNECT: + gu_throw_fatal << "Network reactor termination was requested by " + "WSREP_NODE_ISOLATION_FORCE_DISCONNECT"; + } + return true; /* to keep compiler happy */ +} gu::AsioStreamReact::AsioStreamReact( AsioIoService& io_service, @@ -33,6 +47,7 @@ , local_addr_() , remote_addr_() , connected_() + , handshake_complete_() , non_blocking_(false) , in_progress_() , read_context_() @@ -53,7 +68,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "error opening stream socket " << uri; } @@ -63,7 +78,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "error checking if socket is open "; return false; } @@ -81,7 +96,7 @@ // Catch all the possible exceptions here, not only asio ones. catch (const std::exception& e) { - log_warn << "Closing socket failed: " << e.what(); + log_info << "Closing socket failed: " << e.what(); } void gu::AsioStreamReact::bind(const gu::AsioIpAddress& addr) try @@ -90,7 +105,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error in binding"; + gu_throw_system_error(e.code().value()) << "error in binding"; } void gu::AsioStreamReact::async_connect( @@ -112,7 +127,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error connecting "; + gu_throw_system_error(e.code().value()) << "error connecting "; } @@ -126,13 +141,15 @@ { gu_throw_error(EBUSY) << "Trying to write into busy socket"; } - + if (not handshake_complete_) { + gu_throw_error(EBUSY) << "Handshake in progress"; + } write_context_ = WriteContext(bufs); start_async_write(&AsioStreamReact::write_handler, handler); } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Async write failed '" + gu_throw_system_error(e.code().value()) << "Async write failed '" << e.what(); } @@ -143,12 +160,16 @@ GU_ASIO_DEBUG(debug_print() << " AsioStreamReact::async_read: buf pointer: " << buf.data() << " buf size: " << buf.size()); assert(not read_context_.buf().data()); + if (not handshake_complete_) { + gu_throw_error(EBUSY) << "Handshake in progress"; + } + assert(handshake_complete_); read_context_ = ReadContext(buf); start_async_read(&AsioStreamReact::read_handler, handler); } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Async read failed '" + gu_throw_system_error(e.code().value()) << "Async read failed '" << e.what(); } @@ -157,7 +178,7 @@ { auto last_error(engine.last_error()); if (last_error.is_system()) - gu_throw_error(last_error.value()) << prefix + gu_throw_system_error(last_error.value()) << prefix << ": " << last_error.message(); else gu_throw_error(EPROTO) << prefix @@ -194,7 +215,7 @@ } catch (asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to connect '" + gu_throw_system_error(e.code().value()) << "Failed to connect '" << uri << "': " << e.what(); } @@ -221,7 +242,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to write: " << e.what(); + gu_throw_system_error(e.code().value()) << "Failed to write: " << e.what(); } size_t gu::AsioStreamReact::read(const AsioMutableBuffer& buf) try @@ -256,7 +277,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to read: " << e.what(); + gu_throw_system_error(e.code().value()) << "Failed to read: " << e.what(); } std::string gu::AsioStreamReact::local_addr() const @@ -276,7 +297,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error setting receive buffer size"; + gu_throw_system_error(e.code().value()) << "error setting receive buffer size"; } size_t gu::AsioStreamReact::get_receive_buffer_size() try @@ -285,7 +306,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error getting receive buffer size "; + gu_throw_system_error(e.code().value()) << "error getting receive buffer size "; } void gu::AsioStreamReact::set_send_buffer_size(size_t size) try @@ -295,7 +316,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error setting send buffer size"; + gu_throw_system_error(e.code().value()) << "error setting send buffer size"; } size_t gu::AsioStreamReact::get_send_buffer_size() try @@ -304,7 +325,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error getting send buffer size"; + gu_throw_system_error(e.code().value()) << "error getting send buffer size"; } struct tcp_info gu::AsioStreamReact::get_tcp_info() try @@ -313,7 +334,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error getting TCP info"; + gu_throw_system_error(e.code().value()) << "error getting TCP info"; } @@ -321,9 +342,11 @@ const std::shared_ptr& handler, AsioStreamEngine::op_status result) try { + GU_ASIO_DEBUG(debug_print() << " complete_client_handshake " << result); switch (result) { case AsioStreamEngine::success: + handshake_complete_ = true; handler->connect_handler(*this, AsioErrorCode()); break; case AsioStreamEngine::want_read: @@ -403,6 +426,13 @@ close(); return; } + + if (is_isolated()) + { + handle_isolation_error(handler); + return; + } + auto result(engine_->client_handshake()); GU_ASIO_DEBUG(debug_print() << "AsioStreamReact::client_handshake_handler: result from engine: " @@ -410,6 +440,7 @@ switch (result) { case AsioStreamEngine::success: + handshake_complete_ = true; handler->connect_handler( *this, AsioErrorCode(ec.value(), ec.category())); break; @@ -439,54 +470,40 @@ } void gu::AsioStreamReact::complete_server_handshake( - const std::shared_ptr& acceptor, - AsioStreamEngine::op_status result, - const std::shared_ptr& acceptor_handler) try + const std::shared_ptr& handler, + AsioStreamEngine::op_status result) try { GU_ASIO_DEBUG(debug_print() << "AsioStreamReact::server_handshake_handler: " << "result from engine: " << result); switch (result) { case AsioStreamEngine::success: - acceptor_handler->accept_handler(*acceptor, shared_from_this(), - AsioErrorCode()); + handshake_complete_ = true; + handler->connect_handler(*this, AsioErrorCode()); break; case AsioStreamEngine::want_read: start_async_read(&AsioStreamReact::server_handshake_handler, - acceptor, - acceptor_handler); + handler); break; case AsioStreamEngine::want_write: start_async_write(&AsioStreamReact::server_handshake_handler, - acceptor, - acceptor_handler); + handler); break; case AsioStreamEngine::error: - log_warn << "Handshake failed: " << engine_->last_error(); - // Fall through + handler->connect_handler(*this, engine_->last_error()); + break; case AsioStreamEngine::eof: - // Restart accepting transparently. The socket will go out of - // scope and will be destructed. - // - // However, note that with this way of notifying the initiator - // of accept operation will never happen before the handshake - // is over. This means that there may be only one socket performing - // server side handshake at the time. To get around this, the - // actual connect/accept events must be exposed to acceptor/connector - // handler, forcing them to initiate handshake. - acceptor->async_accept(acceptor_handler); + handler->connect_handler(*this, AsioErrorCode::make_eof()); break; } } catch (const asio::system_error& e) { - acceptor_handler->accept_handler(*acceptor, shared_from_this(), - AsioErrorCode(e.code().value())); + handler->connect_handler(*this, AsioErrorCode(e.code().value())); } void gu::AsioStreamReact::server_handshake_handler( - const std::shared_ptr& acceptor, - const std::shared_ptr& acceptor_handler, + const std::shared_ptr& handler, const asio::error_code& ec) try { // During handshake there is only read or write in progress @@ -494,25 +511,28 @@ in_progress_ &= ~(read_in_progress | write_in_progress); if (ec) { - acceptor_handler->accept_handler( - *acceptor, shared_from_this(), - AsioErrorCode(ec.value(), ec.category())); + handler->connect_handler(*this, + AsioErrorCode(ec.value(), ec.category())); return; } + if (is_isolated()) + { + throw asio::system_error(asio::error::basic_errors::operation_aborted); + } + auto result = engine_->server_handshake(); auto self = shared_from_this(); // Clear possible write IO in_progress_ &= write_in_progress; - socket_.async_wait(socket_.wait_write, [acceptor, acceptor_handler, result, + socket_.async_wait(socket_.wait_write, [handler, result, self](const asio::error_code& ec) { - self->complete_server_handshake(acceptor, result, acceptor_handler); + self->complete_server_handshake(handler, result); }); } catch (const asio::system_error& e) { - acceptor_handler->accept_handler(*acceptor, shared_from_this(), - AsioErrorCode(e.code().value())); + handler->connect_handler(*this, AsioErrorCode(e.code().value())); } void gu::AsioStreamReact::read_handler( @@ -531,6 +551,12 @@ return; } + if (is_isolated()) + { + handle_isolation_error(handler); + return; + } + const size_t left_to_read(read_context_.left_to_read()); assert(left_to_read <= read_context_.buf().size() - read_context_.bytes_transferred()); @@ -596,6 +622,12 @@ return; } + if (is_isolated()) + { + handle_isolation_error(handler); + return; + } + AsioStreamEngine::op_result write_result( engine_->write( write_context_.buf().data() + write_context_.bytes_transferred(), @@ -710,7 +742,7 @@ read_context_.bytes_transferred())); if (read_completion == 0) { - auto total_transferred(read_context_.bytes_transferred()); + std::size_t total_transferred(read_context_.bytes_transferred()); read_context_.reset(); handler->read_handler(*this, AsioErrorCode(), total_transferred); } @@ -735,7 +767,7 @@ write_context_.inc_bytes_transferred(bytes_transferred); if (write_context_.bytes_transferred() == write_context_.buf().size()) { - auto total_transferred(write_context_.bytes_transferred()); + std::size_t total_transferred(write_context_.bytes_transferred()); write_context_.reset(); handler->write_handler(*this, AsioErrorCode(), total_transferred); } @@ -774,6 +806,18 @@ close(); } +void gu::AsioStreamReact::handle_isolation_error( + const std::shared_ptr& handler) +{ + shutdown(); + handler->write_handler( + *this, + AsioErrorCode(asio::error::basic_errors::operation_aborted, + asio::error::get_system_category()), + 0); + close(); +} + void gu::AsioStreamReact::set_non_blocking(bool val) { // Socket which is once set to non-blocking mode should never @@ -827,7 +871,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to open acceptor: " << e.what(); + gu_throw_system_error(e.code().value()) << "Failed to open acceptor: " << e.what(); } @@ -847,7 +891,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to listen: " << e.what(); + gu_throw_system_error(e.code().value()) << "Failed to listen: " << e.what(); } void gu::AsioAcceptorReact::close() try @@ -860,29 +904,28 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to close acceptor: " + gu_throw_system_error(e.code().value()) << "Failed to close acceptor: " << e.what(); } void gu::AsioAcceptorReact::async_accept( - const std::shared_ptr& handler, + const std::shared_ptr& acceptor_handler, + const std::shared_ptr& handler, const std::shared_ptr& engine) try { GU_ASIO_DEBUG(this << " AsioAcceptorReact::async_accept: " << listen_addr()); auto new_socket(std::make_shared( io_service_, scheme_, engine)); - acceptor_.async_accept(new_socket->socket_, - boost::bind(&AsioAcceptorReact::accept_handler, - shared_from_this(), - new_socket, - handler, - asio::placeholders::error)); - + auto self = shared_from_this(); + acceptor_.async_accept( + new_socket->socket_, [self, new_socket, acceptor_handler, + handler](const asio::error_code& ec) + { self->accept_handler(new_socket, acceptor_handler, handler, ec); }); } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to accept: " << e.what(); + gu_throw_system_error(e.code().value()) << "Failed to accept: " << e.what(); } @@ -922,7 +965,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "Failed to accept: " << e.what(); + gu_throw_system_error(e.code().value()) << "Failed to accept: " << e.what(); } std::string gu::AsioAcceptorReact::listen_addr() const try @@ -934,7 +977,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "failed to read listen addr " << "', asio error '" << e.what() << "'"; } @@ -945,7 +988,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) + gu_throw_system_error(e.code().value()) << "failed to read listen port " << "', asio error '" << e.what() << "'"; } @@ -957,7 +1000,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error setting receive buffer size"; + gu_throw_system_error(e.code().value()) << "error setting receive buffer size"; } @@ -967,7 +1010,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error getting receive buffer size"; + gu_throw_system_error(e.code().value()) << "error getting receive buffer size"; return 0; } @@ -978,7 +1021,7 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error setting send buffer size"; + gu_throw_system_error(e.code().value()) << "error setting send buffer size"; } size_t gu::AsioAcceptorReact::get_send_buffer_size() try @@ -987,19 +1030,20 @@ } catch (const asio::system_error& e) { - gu_throw_error(e.code().value()) << "error getting send buffer size"; + gu_throw_system_error(e.code().value()) << "error getting send buffer size"; return 0; } void gu::AsioAcceptorReact::accept_handler( const std::shared_ptr& socket, - const std::shared_ptr& handler, + const std::shared_ptr& acceptor_handler, + const std::shared_ptr& handler, const asio::error_code& ec) try { GU_ASIO_DEBUG(this << " AsioAcceptorReact::accept_handler(): " << ec); if (ec) { - handler->accept_handler( + acceptor_handler->accept_handler( *this, socket, AsioErrorCode(ec.value(), ec.category())); return; } @@ -1010,22 +1054,23 @@ socket->assign_addresses(); std::string remote_ip = gu::unescape_addr(::escape_addr(socket->socket_.remote_endpoint().address())); - auto connection_allowed(gu::allowlist_value_check(WSREP_ALLOWLIST_KEY_IP, remote_ip)); + bool connection_allowed(gu::allowlist_value_check(WSREP_ALLOWLIST_KEY_IP, remote_ip)); if (connection_allowed == false) { - log_warn << "Connection not allowed, IP " << remote_ip << " not found in allowlist."; - async_accept(handler); + log_warn << "Connection not allowed, IP " << + remote_ip << " not found in allowlist."; + acceptor_handler->accept_handler(*this, socket, AsioErrorCode::make_eof()); return; } socket->connected_ = true; // Necessary async reads/writes/waits are done within // server_handshake_handler(). - socket->server_handshake_handler(shared_from_this(), handler, ec); + acceptor_handler->accept_handler(*this, socket, AsioErrorCode()); + socket->server_handshake_handler(handler, ec); } catch(const asio::system_error& e) { - log_warn << "Failed to accept new connection: '" << e.what() << "'"; - async_accept(handler); - return; + acceptor_handler->accept_handler(*this, socket, + AsioErrorCode(e.code().value())); } diff -Nru galera-4-26.4.18/galerautils/src/gu_asio_stream_react.hpp galera-4-26.4.20/galerautils/src/gu_asio_stream_react.hpp --- galera-4-26.4.18/galerautils/src/gu_asio_stream_react.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_asio_stream_react.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ // -// Copyright (C) 2020 Codership Oy +// Copyright (C) 2020-2024 Codership Oy // /** @file gu_asio_stream_react.hpp @@ -74,16 +74,14 @@ const std::shared_ptr&, AsioStreamEngine::op_status); void complete_server_handshake( - const std::shared_ptr&, - AsioStreamEngine::op_status, - const std::shared_ptr&); + const std::shared_ptr&, + AsioStreamEngine::op_status); void connect_handler(const std::shared_ptr&, const asio::error_code& ec); void client_handshake_handler(const std::shared_ptr&, const asio::error_code&); void server_handshake_handler( - const std::shared_ptr& acceptor, - const std::shared_ptr& acceptor_handler, + const std::shared_ptr&, const asio::error_code& ec); void read_handler(const std::shared_ptr&, const asio::error_code&); @@ -113,6 +111,8 @@ void handle_write_handler_error( const std::shared_ptr&, const AsioErrorCode&); + void handle_isolation_error( + const std::shared_ptr&); void set_non_blocking(bool); @@ -127,6 +127,7 @@ std::string local_addr_; std::string remote_addr_; bool connected_; + bool handshake_complete_; bool non_blocking_; // Flags and state for operations in progress. @@ -242,6 +243,7 @@ virtual void close() GALERA_OVERRIDE; virtual void async_accept( const std::shared_ptr&, + const std::shared_ptr&, const std::shared_ptr& engine = nullptr) GALERA_OVERRIDE; virtual std::shared_ptr accept() GALERA_OVERRIDE; @@ -255,6 +257,7 @@ // ASIO handlers void accept_handler(const std::shared_ptr&, const std::shared_ptr&, + const std::shared_ptr&, const asio::error_code&); private: std::string debug_print() const; @@ -264,7 +267,7 @@ bool listening_; std::shared_ptr engine_; }; -} +} // namespace gu #include "gu_enable_non_virtual_dtor.hpp" diff -Nru galera-4-26.4.18/galerautils/src/gu_datetime.cpp galera-4-26.4.20/galerautils/src/gu_datetime.cpp --- galera-4-26.4.18/galerautils/src/gu_datetime.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_datetime.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -80,7 +80,7 @@ const auto& str = parts[RealParts::decimal].str(); if (str.size()) { - const auto n_decis = str.size(); + const size_t n_decis = str.size(); if (n_decis > 9) { throw gu::NotFound(); @@ -101,7 +101,7 @@ long long seconds_from_string(const std::string& str) { auto real = real_from_string(str); - const auto max = std::numeric_limits::max(); + const long long max = std::numeric_limits::max(); if (max/gu::datetime::Sec < real.integer) { /* Multiplication would overflow */ @@ -120,8 +120,8 @@ template long long seconds_from_string_mult(const std::string& str) try { - const auto val = std::stoll(str); - const auto max = std::numeric_limits::max(); + const long long val = std::stoll(str); + const long long max = std::numeric_limits::max(); if (max/Mult < val) { /* Multiplication would overflow */ diff -Nru galera-4-26.4.18/galerautils/src/gu_fdesc.cpp galera-4-26.4.20/galerautils/src/gu_fdesc.cpp --- galera-4-26.4.18/galerautils/src/gu_fdesc.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_fdesc.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -124,7 +124,7 @@ if (ftruncate(fd_, size_)) { - gu_throw_error(errno) << "Failed to truncate '" << name_ + gu_throw_system_error(errno) << "Failed to truncate '" << name_ << "' to " << size_ << " bytes."; } } @@ -138,7 +138,8 @@ FileDescriptor::constructor_common() { if (fd_ < 0) { - gu_throw_error(errno) << "Failed to open file '" + name_ + '\''; + gu_throw_system_error(errno) + << "Failed to open file '" + name_ + '\''; } #if !defined(__APPLE__) /* Darwin does not have posix_fadvise */ /* benefits are questionable @@ -180,7 +181,7 @@ log_debug << "Flushing file '" << name_ << "'"; if (fsync (fd_) < 0) { - gu_throw_error(errno) << "fsync() failed on '" + name_ + '\''; + gu_throw_system_error(errno) << "fsync() failed on '" + name_ + '\''; } log_debug << "Flushed file '" << name_ << "'"; @@ -192,10 +193,12 @@ byte_t const byte (0); if (lseek (fd_, offset, SEEK_SET) != offset) - gu_throw_error(errno) << "lseek() failed on '" << name_ << '\''; + gu_throw_system_error(errno) + << "lseek() failed on '" << name_ << '\''; if (write (fd_, &byte, sizeof(byte)) != sizeof(byte)) - gu_throw_error(errno) << "write() failed on '" << name_ << '\''; + gu_throw_system_error(errno) + << "write() failed on '" << name_ << '\''; return true; } @@ -221,7 +224,7 @@ return; } - gu_throw_error (errno) << "File preallocation failed"; + gu_throw_system_error (errno) << "File preallocation failed"; } void @@ -248,7 +251,7 @@ } else { - gu_throw_error (errno) << "File preallocation failed"; + gu_throw_system_error (errno) << "File preallocation failed"; } } } diff -Nru galera-4-26.4.18/galerautils/src/gu_fifo.c galera-4-26.4.20/galerautils/src/gu_fifo.c --- galera-4-26.4.18/galerautils/src/gu_fifo.c 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_fifo.c 2024-07-30 05:28:41.000000000 +0000 @@ -111,7 +111,7 @@ if (max_size > gu_avphys_bytes()) { gu_error ("Maximum FIFO size %llu exceeds available memory " - "limit %llu", max_size, gu_avphys_bytes()); + "limit %zu", max_size, gu_avphys_bytes()); return NULL; } @@ -122,8 +122,8 @@ } - gu_debug ("Creating FIFO buffer of %llu elements of size %llu, " - "memory min used: %zu, max used: %zu", + gu_debug ("Creating FIFO buffer of %llu elements of size %zu, " + "memory min used: %llu, max used: %llu", array_len * row_len, item_size, alloc_size, alloc_size + array_len*row_size); @@ -143,7 +143,7 @@ gu_cond_init (&ret->put_cond, NULL); } else { - gu_error ("Failed to allocate %zu bytes for FIFO", alloc_size); + gu_error ("Failed to allocate %llu bytes for FIFO", alloc_size); } } @@ -204,7 +204,7 @@ /* if there are items in the queue, wait until they are all fetched */ while (q->used > 0 && 0 == ret) { /* will make getters to signal every time item is removed */ - gu_warn ("Waiting for %lu items to be fetched.", q->used); + gu_warn ("Waiting for %u items to be fetched.", q->used); q->put_wait++; ret = gu_cond_wait (&q->put_cond, &q->lock); } diff -Nru galera-4-26.4.18/galerautils/src/gu_init.c galera-4-26.4.20/galerautils/src/gu_init.c --- galera-4-26.4.18/galerautils/src/gu_init.c 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_init.c 2024-07-30 05:28:41.000000000 +0000 @@ -18,7 +18,7 @@ size_t const page_size = GU_PAGE_SIZE; if (page_size & (page_size - 1)) { - gu_fatal("GU_PAGE_SIZE(%z) is not a power of 2", GU_PAGE_SIZE); + gu_fatal("GU_PAGE_SIZE(%zu) is not a power of 2", GU_PAGE_SIZE); gu_abort(); } diff -Nru galera-4-26.4.18/galerautils/src/gu_lock.hpp galera-4-26.4.20/galerautils/src/gu_lock.hpp --- galera-4-26.4.18/galerautils/src/gu_lock.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_lock.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -68,7 +68,7 @@ mtx_.owned_ = gu_thread_self(); #endif /* GU_MUTEX_DEBUG */ - if (gu_unlikely(ret)) gu_throw_error(ret); + if (gu_unlikely(ret)) gu_throw_system_error(ret); } }; } diff -Nru galera-4-26.4.18/galerautils/src/gu_log.c galera-4-26.4.20/galerautils/src/gu_log.c --- galera-4-26.4.18/galerautils/src/gu_log.c 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_log.c 2024-07-30 05:28:41.000000000 +0000 @@ -133,6 +133,7 @@ const char* file, const char* function, const int line, + const char* fmt, ...) { va_list ap; @@ -163,12 +164,10 @@ str += len; max_string -= len; - va_start (ap, line); + va_start (ap, fmt); { - const char* format = va_arg (ap, const char*); - - if (gu_likely(max_string > 0 && NULL != format)) { - vsnprintf (str, max_string, format, ap); + if (gu_likely(max_string > 0 && NULL != fmt)) { + vsnprintf (str, max_string, fmt, ap); } } va_end (ap); diff -Nru galera-4-26.4.18/galerautils/src/gu_log.h galera-4-26.4.20/galerautils/src/gu_log.h --- galera-4-26.4.18/galerautils/src/gu_log.h 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_log.h 2024-07-30 05:28:41.000000000 +0000 @@ -54,7 +54,8 @@ const char* file, const char* function, const int line, - ...); + const char* fmt, + ...) __attribute__((format(printf, 5, 6))); /** This variable is made global only for the purpose of using it in * gu_debug() macro and avoid calling gu_log() when debug is off. @@ -68,23 +69,25 @@ #endif #if !defined(__cplusplus) || defined(GALERA_LOG_H_ENABLE_CXX) -// NOTE: don't add "\n" here even if you really want to do it -#define GU_LOG_C(level, ...)\ - gu_log(level, __FILE__, __func__, __LINE__,\ - __VA_ARGS__, NULL) - /** * @name Logging macros. * Must be implemented as macros to report the location of the code where * they are called. */ /*@{*/ -#define gu_fatal(...) GU_LOG_C(GU_LOG_FATAL, __VA_ARGS__, NULL) -#define gu_error(...) GU_LOG_C(GU_LOG_ERROR, __VA_ARGS__, NULL) -#define gu_warn(...) GU_LOG_C(GU_LOG_WARN, __VA_ARGS__, NULL) -#define gu_info(...) GU_LOG_C(GU_LOG_INFO, __VA_ARGS__, NULL) -#define gu_debug(...) if (gu_unlikely(gu_log_debug)) \ - { GU_LOG_C(GU_LOG_DEBUG, __VA_ARGS__, NULL); } +#define gu_fatal(...) \ + gu_log(GU_LOG_FATAL, __FILE__, __func__, __LINE__, __VA_ARGS__); +#define gu_error(...) \ + gu_log(GU_LOG_ERROR, __FILE__, __func__, __LINE__, __VA_ARGS__); +#define gu_warn(...) \ + gu_log(GU_LOG_WARN, __FILE__, __func__, __LINE__, __VA_ARGS__); +#define gu_info(...) \ + gu_log(GU_LOG_INFO, __FILE__, __func__, __LINE__, __VA_ARGS__) +#define gu_debug(...) \ + if (gu_unlikely(gu_log_debug)) \ + { \ + gu_log(GU_LOG_DEBUG, __FILE__, __func__, __LINE__, __VA_ARGS__); \ + } /*@}*/ #endif /* __cplusplus */ diff -Nru galera-4-26.4.18/galerautils/src/gu_mmap.cpp galera-4-26.4.20/galerautils/src/gu_mmap.cpp --- galera-4-26.4.18/galerautils/src/gu_mmap.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_mmap.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -38,8 +38,8 @@ { if (!mapped) { - gu_throw_error(errno) << "mmap() on '" << fd.name() - << "' failed"; + gu_throw_system_error(errno) + << "mmap() on '" << fd.name() << "' failed"; } #if defined(MADV_DONTFORK) @@ -93,8 +93,8 @@ if (::msync(sync_addr, sync_length, MS_SYNC) < 0) { - gu_throw_error(errno) << "msync(" << sync_addr << ", " - << sync_length << ") failed"; + gu_throw_system_error(errno) + << "msync(" << sync_addr << ", " << sync_length << ") failed"; } } @@ -110,8 +110,8 @@ { if (munmap (ptr, size) < 0) { - gu_throw_error(errno) << "munmap(" << ptr << ", " << size - << ") failed"; + gu_throw_system_error(errno) + << "munmap(" << ptr << ", " << size << ") failed"; } mapped = false; diff -Nru galera-4-26.4.18/galerautils/src/gu_mutex.hpp galera-4-26.4.20/galerautils/src/gu_mutex.hpp --- galera-4-26.4.18/galerautils/src/gu_mutex.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_mutex.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -41,7 +41,7 @@ if (gu_unlikely(err != 0)) { assert(0); - gu_throw_error(err) << "gu_mutex_destroy()"; + gu_throw_system_error(err) << "gu_mutex_destroy()"; } } @@ -58,7 +58,7 @@ else { assert(0); - gu_throw_error(err) << "Mutex lock failed"; + gu_throw_system_error(err) << "Mutex lock failed"; } } diff -Nru galera-4-26.4.18/galerautils/src/gu_resolver.cpp galera-4-26.4.20/galerautils/src/gu_resolver.cpp --- galera-4-26.4.18/galerautils/src/gu_resolver.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_resolver.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -246,7 +246,7 @@ if (fd == -1) { err = errno; - gu_throw_error(err) << "could not create socket"; + gu_throw_system_error(err) << "could not create socket"; } if ((err = ioctl(fd, GU_SIOCGIFCONF, &ifc)) == -1) { @@ -290,7 +290,7 @@ #endif /* !__APPLE__ && !__FreeBSD__ */ if (err != 0) { - gu_throw_error(err) << "failed to get interface index"; + gu_throw_system_error(err) << "failed to get interface index"; } else { @@ -465,7 +465,7 @@ if (inet_ntop(get_family(), addr.get_addr(), dst, sizeof(dst)) == 0) { - gu_throw_error(errno) << "inet ntop failed"; + gu_throw_system_error(errno) << "inet ntop failed"; } switch (get_family()) diff -Nru galera-4-26.4.18/galerautils/src/gu_thread.cpp galera-4-26.4.20/galerautils/src/gu_thread.cpp --- galera-4-26.4.18/galerautils/src/gu_thread.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_thread.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -74,7 +74,7 @@ int err; if ((err = pthread_getschedparam(thd, &policy, &sp)) != 0) { - gu_throw_error(err) << "Failed to read thread schedparams"; + gu_throw_system_error(err) << "Failed to read thread schedparams"; } return ThreadSchedparam(policy, sp.sched_priority); } @@ -102,7 +102,8 @@ } else { - gu_throw_error(err) << "Failed to set thread schedparams " << sp; + gu_throw_system_error(err) + << "Failed to set thread schedparams " << sp; } } } diff -Nru galera-4-26.4.18/galerautils/src/gu_throw.hpp galera-4-26.4.20/galerautils/src/gu_throw.hpp --- galera-4-26.4.18/galerautils/src/gu_throw.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_throw.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -46,6 +46,7 @@ ThrowBase& operator= (const ThrowBase&); friend class ThrowError; + friend class ThrowSystemError; friend class ThrowFatal; }; @@ -64,7 +65,38 @@ ~ThrowError() GU_NOEXCEPT(false) GU_NORETURN { - base.os << ": " << err << " (" << ::strerror(err) << ')'; + Exception e(base.os.str(), err); + + e.trace (base.file, base.func, base.line); + // cppcheck-suppress exceptThrowInDestructor + throw e; + } + + std::ostringstream& msg () { return base.os; } + + private: + + ThrowBase base; + int const err; + }; + + /* final */ class ThrowSystemError + { + public: + + ThrowSystemError (const char* file_, + const char* func_, + int line_, + int err_) + : + base (file_, func_, line_), + err (err_) + {} + + ~ThrowSystemError() GU_NOEXCEPT(false) GU_NORETURN + { + base.os << ": System error: " << err << " (" << ::strerror(err) + << ')'; Exception e(base.os.str(), err); @@ -114,6 +146,9 @@ #define gu_throw_error(err_) \ gu::ThrowError(__FILE__, __FUNCTION__, __LINE__, err_).msg() +#define gu_throw_system_error(err_) \ + gu::ThrowSystemError(__FILE__, __FUNCTION__, __LINE__, err_).msg() + #define gu_throw_fatal \ gu::ThrowFatal(__FILE__, __FUNCTION__, __LINE__).msg() diff -Nru galera-4-26.4.18/galerautils/src/gu_to.c galera-4-26.4.20/galerautils/src/gu_to.c --- galera-4-26.4.18/galerautils/src/gu_to.c 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/src/gu_to.c 2024-07-30 05:28:41.000000000 +0000 @@ -12,6 +12,8 @@ * section is required, these functions can be used to do this. */ + +#include #include #include #include @@ -130,12 +132,12 @@ #ifdef TO_USE_SIGNAL if (gu_cond_destroy (&w->cond)) { // @todo: what if someone is waiting? - gu_warn ("Failed to destroy condition %d. Should not happen", i); + gu_warn ("Failed to destroy condition %zd. Should not happen", i); } #else if (pthread_mutex_destroy (&w->mtx)) { // @todo: what if someone is waiting? - gu_warn ("Failed to destroy mutex %d. Should not happen", i); + gu_warn ("Failed to destroy mutex %zd. Should not happen", i); } #endif } @@ -160,7 +162,7 @@ assert (seqno >= 0); if ((err = gu_mutex_lock(&to->lock))) { - gu_fatal("Mutex lock failed (%d): %s", err, strerror(err)); + gu_fatal("Mutex lock failed (%ld): %s", err, strerror(err)); abort(); } @@ -220,7 +222,8 @@ err = -ECANCELED; break; default: - gu_fatal("Invalid cond wait exit state %d, seqno %llu(%llu)", + gu_fatal("Invalid cond wait exit state %d, seqno %" PRId64 + "(%" PRId64 ")", w->state, seqno, to->seqno); abort(); } @@ -247,7 +250,7 @@ err = pthread_mutex_unlock (&w->mtx); #endif if (err) { - gu_fatal ("gu_cond_signal failed: %d", err); + gu_fatal ("gu_cond_signal failed: %ld", err); } } return err; @@ -277,7 +280,7 @@ assert (seqno >= 0); if ((err = gu_mutex_lock(&to->lock))) { - gu_fatal("Mutex lock failed (%d): %s", err, strerror(err)); + gu_fatal("Mutex lock failed (%ld): %s", err, strerror(err)); abort(); } @@ -321,7 +324,7 @@ assert (seqno >= 0); if ((err = gu_mutex_lock (&to->lock))) { - gu_fatal("Mutex lock failed (%d): %s", err, strerror(err)); + gu_fatal("Mutex lock failed (%ld): %s", err, strerror(err)); abort(); } @@ -337,15 +340,19 @@ err = to_wake_waiter (w); w->state = CANCELED; } else if (seqno == to->seqno && w->state == HOLDER) { - gu_warn("tried to cancel current TO holder, state %d seqno %llu", - w->state, seqno); + gu_warn("tried to cancel current TO holder, state %d seqno %" PRId64, + w->state, seqno); err = -ECANCELED; - } else { - gu_warn("trying to cancel used seqno: state %d cancel seqno = %llu, " - "TO seqno = %llu", w->state, seqno, to->seqno); - err = -ECANCELED; } - + else + { + gu_warn("trying to cancel used seqno: state %d cancel seqno = %" PRId64 + ", " + "TO seqno = %" PRId64, + w->state, seqno, to->seqno); + err = -ECANCELED; + } + gu_mutex_unlock (&to->lock); return err; } @@ -358,7 +365,7 @@ assert (seqno >= 0); if ((err = gu_mutex_lock (&to->lock))) { - gu_fatal("Mutex lock failed (%d): %s", err, strerror(err)); + gu_fatal("Mutex lock failed (%ld): %s", err, strerror(err)); abort(); } @@ -394,7 +401,7 @@ assert (seqno >= 0); if ((err = gu_mutex_lock (&to->lock))) { - gu_fatal("Mutex lock failed (%d): %s", err, strerror(err)); + gu_fatal("Mutex lock failed (%ld): %s", err, strerror(err)); abort(); } if (seqno >= to->seqno) { @@ -406,33 +413,36 @@ switch (w->state) { case HOLDER: - gu_debug ("trying to interrupt in use seqno: seqno = %llu, " - "TO seqno = %llu", seqno, to->seqno); + gu_debug("trying to interrupt in use seqno: seqno = %" PRId64 ", " + "TO seqno = %" PRId64, + seqno, to->seqno); /* gu_mutex_unlock (&to->lock); */ rcode = -ERANGE; break; case CANCELED: - gu_debug ("trying to interrupt canceled seqno: seqno = %llu, " - "TO seqno = %llu", seqno, to->seqno); + gu_debug("trying to interrupt canceled seqno: seqno = %" PRId64 ", " + "TO seqno = %" PRId64, + seqno, to->seqno); /* gu_mutex_unlock (&to->lock); */ rcode = -ERANGE; break; case WAIT: - gu_debug ("signaling to interrupt wait seqno: seqno = %llu, " - "TO seqno = %llu", seqno, to->seqno); - rcode = to_wake_waiter (w); + gu_debug("signaling to interrupt wait seqno: seqno = %" PRId64 ", " + "TO seqno = %" PRId64, + seqno, to->seqno); + rcode = to_wake_waiter(w); /* fall through */ - case RELEASED: - w->state = INTERRUPTED; - break; + case RELEASED: w->state = INTERRUPTED; break; case INTERRUPTED: - gu_debug ("TO waiter interrupt already seqno: seqno = %llu, " - "TO seqno = %llu", seqno, to->seqno); + gu_debug("TO waiter interrupt already seqno: seqno = %" PRId64 ", " + "TO seqno = %" PRId64, + seqno, to->seqno); break; } } else { - gu_debug ("trying to interrupt used seqno: cancel seqno = %llu, " - "TO seqno = %llu", seqno, to->seqno); + gu_debug("trying to interrupt used seqno: cancel seqno = %" PRId64 ", " + "TO seqno = %" PRId64, + seqno, to->seqno); /* gu_mutex_unlock (&to->lock); */ rcode = -ERANGE; } diff -Nru galera-4-26.4.18/galerautils/tests/crc32c_bench.cpp galera-4-26.4.20/galerautils/tests/crc32c_bench.cpp --- galera-4-26.4.18/galerautils/tests/crc32c_bench.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/tests/crc32c_bench.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -77,14 +77,14 @@ } #if __cplusplus >= 201103L - auto start(std::chrono::steady_clock::now()); - auto result(run_bench(len, reps)); - auto stop(std::chrono::steady_clock::now()); - auto duration(std::chrono::duration(stop - start).count()); + auto const start(std::chrono::steady_clock::now()); + uint32_t const result(run_bench(len, reps)); + auto const stop(std::chrono::steady_clock::now()); + double const duration(std::chrono::duration(stop - start).count()); #else struct timeval start, stop; gettimeofday(&start, NULL); - uint32_t result(run_bench(len, reps)); + uint32_t const result(run_bench(len, reps)); gettimeofday(&stop, NULL); double const duration(time_diff(stop, start)); #endif // C++11 diff -Nru galera-4-26.4.18/galerautils/tests/gu_asio_test.cpp galera-4-26.4.20/galerautils/tests/gu_asio_test.cpp --- galera-4-26.4.18/galerautils/tests/gu_asio_test.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/galerautils/tests/gu_asio_test.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -16,10 +16,122 @@ // // Helper classes // +class MockStreamEngine : public gu::AsioStreamEngine +{ +public: + MockStreamEngine(); + ~MockStreamEngine(); + + std::string scheme() const GALERA_OVERRIDE + { + return "mock"; + }; + void assign_fd(int fd) GALERA_OVERRIDE + { + fd_ = fd; + } + + enum op_status client_handshake() GALERA_OVERRIDE + { + ++count_client_handshake_called; + last_error_ = next_error; + return next_result; + } + + enum op_status server_handshake() GALERA_OVERRIDE + { + ++count_server_handshake_called; + last_error_ = next_error; + log_info << "MockStreamEngine::server_handshake: called " + << count_server_handshake_called + << " next_result: " << next_result; + return next_result; + } + + op_result read(void* buf, size_t max_count) GALERA_OVERRIDE + { + ++count_read_called; + ssize_t read_result(::recv(fd_, buf, max_count, 0)); + return map_return_value(read_result, want_read); + } + + op_result write(const void* buf, size_t count) GALERA_OVERRIDE + { + ++count_write_called; + ssize_t write_result(::send(fd_, buf, count, MSG_NOSIGNAL)); + return map_return_value(write_result, want_write); + } + + void shutdown() GALERA_OVERRIDE { } + + gu::AsioErrorCode last_error() const GALERA_OVERRIDE + { + return last_error_; + } + + op_result map_return_value(ssize_t result, + enum op_status return_on_block) + { + if (next_result != success) + { + last_error_ = next_error; + return {next_result, size_t(result)}; + } + + if (result > 0) + { + return {success, size_t(result)}; + } + else if (result == 0) + { + return {eof, size_t(result)}; + } + else if (errno == EAGAIN || errno == EWOULDBLOCK) + { + last_error_ = errno; + return {return_on_block, size_t(result)}; + } + else + { + last_error_ = next_error; + return {error, size_t(result)}; + } + } + + enum op_status next_result; + int next_error; + size_t count_client_handshake_called; + size_t count_server_handshake_called; + size_t count_read_called; + size_t count_write_called; + +private: + int fd_; + int last_error_; +}; + +MockStreamEngine::MockStreamEngine() + : next_result(success) + , next_error() + , count_client_handshake_called() + , count_server_handshake_called() + , count_read_called() + , count_write_called() + , fd_() + , last_error_() +{ + log_info << "MockStreamEngine"; +} + +MockStreamEngine::~MockStreamEngine() +{ + log_info << "~MockStreamEngine"; +} + class MockSocketHandler : public gu::AsioSocketHandler { public: - MockSocketHandler() + MockSocketHandler(const std::string& context = "") : gu::AsioSocketHandler() , write_buffer_() , read_buffer_() @@ -29,16 +141,21 @@ , bytes_read_() , bytes_written_() , last_error_code_() - { } + , context_(context) + { + + log_info << "MockSocketHandler(" << context_ << ")"; + } ~MockSocketHandler() { - log_info << "~MockSocketHandler()"; + log_info << "~MockSocketHandler(" << context_ << ")"; } virtual void connect_handler(gu::AsioSocket& socket, const gu::AsioErrorCode& ec) GALERA_OVERRIDE { - log_info << "connected: " << &socket; + log_info << "MockSocketHandler(" << context_ << ") connected: " << &socket + << " error_code: " << ec; invocations_.push_back("connect"); connect_handler_called_ = true; last_error_code_ = ec; @@ -100,6 +217,7 @@ size_t bytes_read_; size_t bytes_written_; gu::AsioErrorCode last_error_code_; + std::string context_; }; #include "gu_disable_non_virtual_dtor.hpp" @@ -109,7 +227,10 @@ { public: MockAcceptorHandler() - : accepted_socket_() + : cur_stream_engine() + , next_stream_engine() + , next_socket_handler(std::make_shared("server")) + , accepted_socket_() , accepted_handler_() { } @@ -120,12 +241,18 @@ const std::shared_ptr& socket, const gu::AsioErrorCode& ec) GALERA_OVERRIDE { - log_info << "accepted " << socket.get(); - accepted_socket_ = socket; - accepted_handler_ = std::make_shared(); - // For some reason progress halts if acceptor does not keep - // accepting. - acceptor.async_accept(shared_from_this()); + log_info << "accepted " << socket.get() << " error code: " << ec; + if (not ec) { + accepted_socket_ = socket; + accepted_handler_ = next_socket_handler; + } + if (next_stream_engine) { + cur_stream_engine = next_stream_engine; + next_stream_engine = std::make_shared(); + } + next_socket_handler = std::make_shared(); + acceptor.async_accept(shared_from_this(), next_socket_handler, + next_stream_engine); } std::shared_ptr accepted_socket() const @@ -142,6 +269,15 @@ accepted_socket_.reset(); accepted_handler_.reset(); } + + /* Stream engine which was assigned during previous call to + * accept_handler(). */ + std::shared_ptr cur_stream_engine; + /* Stream engine which will be assigned when the + * accept_handler() is called next time. */ + std::shared_ptr next_stream_engine; + /* Socket handler for the next accepted connection. */ + std::shared_ptr next_socket_handler; private: std::shared_ptr accepted_socket_; std::shared_ptr accepted_handler_; @@ -267,7 +403,7 @@ void test_socket_receive_buffer_size_common(Socket& socket, const gu::URI& uri) { socket.open(uri); - auto default_size(socket.get_receive_buffer_size()); + size_t default_size(socket.get_receive_buffer_size()); socket.set_receive_buffer_size(default_size/2); ck_assert(socket.get_receive_buffer_size() == default_size/2); @@ -314,7 +450,7 @@ void test_socket_send_buffer_size_common(Socket& socket, const gu::URI& uri) { socket.open(uri); - auto default_size(socket.get_send_buffer_size()); + size_t default_size(socket.get_send_buffer_size()); socket.set_send_buffer_size(default_size/2); ck_assert(socket.get_send_buffer_size() == default_size/2); } @@ -434,7 +570,7 @@ gu::URI uri("tcp://127.0.0.1:0"); auto acceptor(io_service.make_acceptor(uri)); acceptor->open(uri); - auto default_size(acceptor->get_receive_buffer_size()); + size_t default_size(acceptor->get_receive_buffer_size()); acceptor->set_receive_buffer_size(default_size/2); ck_assert(acceptor->get_receive_buffer_size() == default_size/2); } @@ -468,12 +604,25 @@ gu::URI uri("tcp://127.0.0.1:0"); auto acceptor(io_service.make_acceptor(uri)); acceptor->open(uri); - auto default_size(acceptor->get_send_buffer_size()); + size_t default_size(acceptor->get_send_buffer_size()); acceptor->set_send_buffer_size(default_size/2); ck_assert(acceptor->get_send_buffer_size() == default_size/2); } END_TEST +void wait_handshake_ready(gu::AsioIoService& io_service, + MockAcceptorHandler& acceptor_handler, + MockSocketHandler& socket_handler) +{ + while (not(acceptor_handler.accepted_socket() + && acceptor_handler.accepted_handler()->connect_handler_called() + && socket_handler.connect_handler_called())) + { + io_service.run_one(); + } +} + + template void test_connect_common(gu::AsioIoService& io_service, Acceptor& acceptor, @@ -483,11 +632,7 @@ auto socket(io_service.make_socket(acceptor.listen_addr())); socket->async_connect(acceptor.listen_addr(), handler); - while (not (acceptor_handler.accepted_socket() && - handler->connect_handler_called())) - { - io_service.run_one(); - } + wait_handshake_ready(io_service, acceptor_handler, *handler); auto accepted_socket(acceptor_handler.accepted_socket()); ck_assert_msg(acceptor.listen_addr() == accepted_socket->local_addr(), @@ -504,7 +649,8 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_connect_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -516,10 +662,12 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_connect_common(io_service, *acceptor, *acceptor_handler); acceptor_handler->reset(); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_connect_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -532,12 +680,7 @@ auto handler(std::make_shared()); auto socket(io_service.make_socket(acceptor.listen_addr())); socket->async_connect(acceptor.listen_addr(), handler); - - while (not (acceptor_handler.accepted_socket() && - handler->connect_handler_called())) - { - io_service.run_one(); - } + wait_handshake_ready(io_service, acceptor_handler, *handler); const char* hdr = "hdr"; const char* data = "data"; @@ -571,7 +714,8 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_async_read_write_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -585,11 +729,7 @@ auto socket(io_service.make_socket(acceptor.listen_addr())); socket->async_connect(acceptor.listen_addr(), handler); - while (not (acceptor_handler.accepted_socket() && - handler->connect_handler_called())) - { - io_service.run_one(); - } + wait_handshake_ready(io_service, acceptor_handler, *handler); const char* hdr("hdr"); gu::Buffer data(1 << 23); @@ -619,8 +759,10 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); - test_async_read_write_large_common(io_service, *acceptor, *acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); + test_async_read_write_large_common(io_service, *acceptor, + *acceptor_handler); } END_TEST @@ -634,11 +776,7 @@ socket->async_connect(acceptor.listen_addr(), handler); mark_point(); - while (not (acceptor_handler.accepted_socket() && - handler->connect_handler_called())) - { - io_service.run_one(); - } + wait_handshake_ready(io_service, acceptor_handler, *handler); const char* hdr("hdr"); gu::Buffer data(10); @@ -708,9 +846,10 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); - test_async_read_write_small_large_common( - io_service, *acceptor, *acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); + test_async_read_write_small_large_common(io_service, *acceptor, + *acceptor_handler); } END_TEST @@ -723,11 +862,7 @@ auto socket(io_service.make_socket(acceptor.listen_addr())); socket->async_connect(acceptor.listen_addr(), handler); - while (not (acceptor_handler.accepted_socket() && - handler->connect_handler_called())) - { - io_service.run_one(); - } + wait_handshake_ready(io_service, acceptor_handler, *handler); const char* hdr = "hdr"; const char* data = "data"; @@ -761,9 +896,10 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); - test_async_read_from_client_write_from_server_common( - io_service, *acceptor, *acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); + test_async_read_from_client_write_from_server_common(io_service, *acceptor, + *acceptor_handler); } END_TEST @@ -808,7 +944,8 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_write_twice_wo_handling_common(io_service, *acceptor, *acceptor_handler); } @@ -822,11 +959,8 @@ auto socket(io_service.make_socket(acceptor.listen_addr())); socket->async_connect(acceptor.listen_addr(), handler); - while (not (acceptor_handler.accepted_socket() && - handler->connect_handler_called())) - { - io_service.run_one(); - } + wait_handshake_ready(io_service, acceptor_handler, *handler); + socket->close(); char readbuf[1]; @@ -847,7 +981,8 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_close_client_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -883,7 +1018,8 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_close_server_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -913,7 +1049,8 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); test_get_tcp_info_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1023,13 +1160,13 @@ fclose(key_file); } -static void set_x509v3_extensions(X509* x509, X509* issuer) +static void set_x509v3_extensions(X509* x509, X509* issuer, bool const is_ca) { auto* conf_bio = BIO_new(BIO_s_mem()); std::string ext{ "[extensions]\n" "authorityKeyIdentifier=keyid,issuer\n" "subjectKeyIdentifier=hash\n" }; - if (!issuer) + if (is_ca) { ext += "basicConstraints=critical,CA:TRUE\n"; } @@ -1054,7 +1191,9 @@ X509V3_CTX ctx; X509V3_set_ctx(&ctx, issuer ? issuer : x509, x509, nullptr, nullptr, 0); X509V3_set_nconf(&ctx, conf); - if (!X509V3_EXT_add_nconf(conf, &ctx, (char *)"extensions", x509)) + char extensions[16]; + ::strncpy(extensions, "extensions", sizeof(extensions)); + if (!X509V3_EXT_add_nconf(conf, &ctx, extensions, x509)) { throw_error("Could not add extension"); } @@ -1062,7 +1201,8 @@ BIO_free(conf_bio); } -static X509* create_x509(EVP_PKEY* pkey, X509* issuer, const char* cn) +static X509* create_x509(EVP_PKEY* pkey, X509* issuer, const char* cn, + bool const is_ca) { auto* x509 = X509_new(); /* According to standard, value 2 means version 3. */ @@ -1096,7 +1236,7 @@ X509_set_issuer_name(x509, X509_get_subject_name(issuer)); } - set_x509v3_extensions(x509, issuer); + set_x509v3_extensions(x509, issuer, is_ca); X509_sign(x509, pkey, EVP_sha256()); @@ -1136,10 +1276,10 @@ { auto* pkey = create_key(); write_key(pkey, "galera_key.pem"); - auto* ca = create_x509(pkey, nullptr, "Galera Root"); + auto* ca = create_x509(pkey, nullptr, "Galera Root", true); write_x509(ca, "galera_ca.pem"); - auto* cert = create_x509(pkey, ca, "Galera Cert"); + auto* cert = create_x509(pkey, ca, "Galera Cert", false); write_x509(cert, "galera_cert.pem"); X509_free(cert); X509_free(ca); @@ -1155,32 +1295,28 @@ Two bundles consisting of intermediate CA and server certificate are created for servers 1 and 2. */ -static void generate_chains() +static void generate_self_signed_chains() { - auto* root_ca_key = create_key(); - auto* root_ca = create_x509(root_ca_key, nullptr, "Galera Root CA"); - auto* int_ca_key = create_key(); - auto* int_ca = create_x509(int_ca_key, root_ca, "Galera Intermediate CA"); - - auto* server_1_key = create_key(); - auto* server_1_cert = create_x509(server_1_key, int_ca, "Galera Server 1"); - auto* server_2_key = create_key(); - auto* server_2_cert = create_x509(server_2_key, int_ca, "Galera Server 2"); + auto* sign_key = create_key(); + auto* root_ca = create_x509(sign_key, nullptr, "Galera Root CA", true); + auto* int_ca + = create_x509(sign_key, root_ca, "Galera Intermediate CA", true); + auto* server_1_cert + = create_x509(sign_key, int_ca, "Galera Server 1", false); + auto* server_2_cert + = create_x509(sign_key, int_ca, "Galera Server 2", false); write_x509(root_ca, "galera-ca.pem"); - write_key(server_1_key, "galera-server-1.key"); + write_key(sign_key, "galera-server-1.key"); write_x509_list({ server_1_cert, int_ca }, "bundle-galera-server-1.pem"); - write_key(server_2_key, "galera-server-2.key"); + write_key(sign_key, "galera-server-2.key"); write_x509_list({ server_2_cert, int_ca }, "bundle-galera-server-2.pem"); X509_free(server_2_cert); - EVP_PKEY_free(server_2_key); X509_free(server_1_cert); - EVP_PKEY_free(server_1_key); X509_free(int_ca); - EVP_PKEY_free(int_ca_key); X509_free(root_ca); - EVP_PKEY_free(root_ca_key); + EVP_PKEY_free(sign_key); } static void generate_certificates() @@ -1192,7 +1328,7 @@ #endif generate_self_signed(); - generate_chains(); + generate_self_signed_chains(); } // @@ -1290,7 +1426,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_connect_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1300,12 +1436,13 @@ gu::AsioIoService io_service(get_ssl_config()); gu::URI uri("ssl://127.0.0.1:0"); auto acceptor_handler(std::make_shared()); + acceptor_handler->next_stream_engine = nullptr; auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_connect_common(io_service, *acceptor, *acceptor_handler); acceptor_handler->reset(); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_connect_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1317,7 +1454,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_async_read_write_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1329,7 +1466,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_async_read_write_large_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1341,7 +1478,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_async_read_write_small_large_common( io_service, *acceptor, *acceptor_handler); } @@ -1354,7 +1491,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_async_read_from_client_write_from_server_common( io_service, *acceptor, *acceptor_handler); } @@ -1367,7 +1504,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_write_twice_wo_handling_common(io_service, *acceptor, *acceptor_handler); } @@ -1380,7 +1517,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_close_client_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1392,7 +1529,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_close_server_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1404,7 +1541,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_get_tcp_info_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1418,7 +1555,7 @@ auto acceptor_handler(std::make_shared()); auto acceptor(io_service.make_acceptor(uri)); acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); test_async_read_write_common(io_service, *acceptor, *acceptor_handler); } END_TEST @@ -1456,21 +1593,25 @@ auto acceptor(server_io_service.make_acceptor(uri)); acceptor->listen(uri); auto acceptor_handler(std::make_shared()); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler); - auto handler(std::make_shared()); + auto handler(std::make_shared("client")); auto socket(client_io_service.make_socket(acceptor->listen_addr())); socket->async_connect(acceptor->listen_addr(), handler); client_io_service.run_one(); // Process async connect server_io_service.run_one(); // Accept client_io_service.run_one(); // Client hello client_io_service.run_one(); // Client hello IO completion - // server_io_service.run_one(); // Server handles - while (acceptor_handler->accepted_socket() != 0) + + while ( + not(handler->connect_handler_called() + && acceptor_handler->accepted_handler()->connect_handler_called())) { client_io_service.poll_one(); server_io_service.poll_one(); } + ck_assert(!handler->last_error_code()); } END_TEST @@ -1488,7 +1629,7 @@ auto acceptor(server_io_service.make_acceptor(uri)); acceptor->listen(uri); auto acceptor_handler(std::make_shared()); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, acceptor_handler->next_socket_handler); auto handler(std::make_shared()); auto socket(client_io_service.make_socket(acceptor->listen_addr())); @@ -1503,7 +1644,6 @@ client_io_service.poll_one(); server_io_service.poll_one(); } - ck_assert(not acceptor_handler->accepted_socket()); ck_assert_msg(handler->last_error_code().message().find( "unable to get local issuer certificate") != std::string::npos, @@ -1519,112 +1659,12 @@ // Wsrep TLS service. // -class MockStreamEngine : public gu::AsioStreamEngine -{ -public: - MockStreamEngine(); - - std::string scheme() const GALERA_OVERRIDE - { - return "mock"; - }; - void assign_fd(int fd) GALERA_OVERRIDE - { - fd_ = fd; - } - - enum op_status client_handshake() GALERA_OVERRIDE - { - ++count_client_handshake_called; - last_error_ = next_error; - return next_result; - } - - enum op_status server_handshake() GALERA_OVERRIDE - { - log_info << "MockWsrepTlsService::server_handshake"; - ++count_server_handshake_called; - last_error_ = next_error; - return next_result; - } - - op_result read(void* buf, size_t max_count) GALERA_OVERRIDE - { - ++count_read_called; - ssize_t read_result(::recv(fd_, buf, max_count, 0)); - return map_return_value(read_result, want_read); - } - - op_result write(const void* buf, size_t count) GALERA_OVERRIDE - { - ++count_write_called; - ssize_t write_result(::send(fd_, buf, count, MSG_NOSIGNAL)); - return map_return_value(write_result, want_write); - } - - void shutdown() GALERA_OVERRIDE { } - - gu::AsioErrorCode last_error() const GALERA_OVERRIDE - { - return last_error_; - } - - op_result map_return_value(ssize_t result, - enum op_status return_on_block) - { - if (next_result != success) - { - last_error_ = next_error; - return {next_result, size_t(result)}; - } - - if (result > 0) - { - return {success, size_t(result)}; - } - else if (result == 0) - { - return {eof, size_t(result)}; - } - else if (errno == EAGAIN || errno == EWOULDBLOCK) - { - last_error_ = errno; - return {return_on_block, size_t(result)}; - } - else - { - last_error_ = next_error; - return {error, size_t(result)}; - } - } - - enum op_status next_result; - int next_error; - size_t count_client_handshake_called; - size_t count_server_handshake_called; - size_t count_read_called; - size_t count_write_called; - -private: - int fd_; - int last_error_; -}; - -MockStreamEngine::MockStreamEngine() - : next_result(success) - , next_error() - , count_client_handshake_called() - , count_server_handshake_called() - , count_read_called() - , count_write_called() - , fd_() - , last_error_() -{ } struct TlsServiceClientTestFixture { gu::AsioIoService server_io_service; std::shared_ptr client_engine; + std::shared_ptr server_engine; gu::AsioIoService client_io_service; gu::URI uri; std::shared_ptr acceptor; @@ -1634,6 +1674,7 @@ TlsServiceClientTestFixture() : server_io_service() , client_engine(std::make_shared()) + , server_engine(std::make_shared()) , client_io_service(gu::Config()) , uri("tcp://127.0.0.1:0") , acceptor(server_io_service.make_acceptor(uri)) @@ -1642,9 +1683,13 @@ , socket_handler(std::make_shared()) { acceptor->listen(uri); - acceptor->async_accept(acceptor_handler); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler, + server_engine); socket->async_connect(acceptor->listen_addr(), socket_handler); - while (not acceptor_handler->accepted_socket()) + while (not( + acceptor_handler->accepted_socket() + && acceptor_handler->accepted_handler()->connect_handler_called())) { server_io_service.run_one(); } @@ -1745,7 +1790,6 @@ struct TlsServiceServerTestFixture { - std::shared_ptr server_engine; gu::AsioIoService server_io_service; gu::AsioIoService client_io_service; gu::URI uri; @@ -1754,17 +1798,30 @@ std::shared_ptr socket; std::shared_ptr socket_handler; TlsServiceServerTestFixture() - : server_engine(std::make_shared()) - , server_io_service(gu::Config()) + : server_io_service(gu::Config()) , client_io_service() , uri("tcp://127.0.0.1:0") , acceptor(server_io_service.make_acceptor(uri)) , acceptor_handler(std::make_shared()) - , socket(client_io_service.make_socket(uri)) - , socket_handler(std::make_shared()) + , socket() + , socket_handler() { acceptor->listen(uri); - acceptor->async_accept(acceptor_handler, server_engine); + /* Override stream engine for tests to be able to do error injection. */ + acceptor_handler->next_stream_engine + = std::make_shared(); + acceptor->async_accept(acceptor_handler, + acceptor_handler->next_socket_handler, + acceptor_handler->next_stream_engine); + run_async_connect(); + } + + void run_async_connect() + { + socket_handler = nullptr; + socket_handler = std::make_shared(); + socket = client_io_service.make_socket( + uri, std::make_shared()); socket->async_connect(acceptor->listen_addr(), socket_handler); client_io_service.run_one(); client_io_service.run_one(); // IO completion @@ -1799,10 +1856,19 @@ START_TEST(test_server_handshake_want_read) { TlsServiceServerTestFixture f; - f.server_engine->next_result = gu::AsioStreamEngine::want_read; + f.acceptor_handler->next_stream_engine->next_result + = gu::AsioStreamEngine::want_read; f.run_server_while( - [&f]() { return f.server_engine->count_server_handshake_called < 1; }); - ck_assert(f.server_engine->count_server_handshake_called == 1); + [&f]() + { + return not f.acceptor_handler->cur_stream_engine + || f.acceptor_handler->cur_stream_engine + ->count_server_handshake_called + < 1; + }); + ck_assert_int_eq( + f.acceptor_handler->cur_stream_engine->count_server_handshake_called, + 1); // Write to connected socket to make accepted socket readable std::array cbs; @@ -1813,28 +1879,34 @@ f.run_client_while( [&f]() { return f.socket_handler->bytes_written() < 4; }); f.run_server_while( - [&f]() { return f.server_engine->count_server_handshake_called < 2; }); + [&f]() { return f.acceptor_handler->cur_stream_engine->count_server_handshake_called < 2; }); } END_TEST START_TEST(test_server_handshake_want_write) { TlsServiceServerTestFixture f; - f.server_engine->next_result = gu::AsioStreamEngine::want_write; + f.acceptor_handler->next_stream_engine->next_result = gu::AsioStreamEngine::want_write; f.run_server_while( - [&f]() { return f.server_engine->count_server_handshake_called < 2; }); + [&f]() + { + return not f.acceptor_handler->cur_stream_engine + || f.acceptor_handler->cur_stream_engine + ->count_server_handshake_called + < 2; + }); } END_TEST START_TEST(test_server_handshake_eof) { TlsServiceServerTestFixture f; - f.server_engine->next_result = gu::AsioStreamEngine::eof; + f.acceptor_handler->next_stream_engine->next_result + = gu::AsioStreamEngine::eof; f.server_io_service.run_one(); - // Acceptor silently discards accepted socket which fails during - // handshake and restarts async accept. - ck_assert(f.acceptor_handler->accepted_socket() == 0); - ck_assert(f.server_engine->count_server_handshake_called == 1); + ck_assert_int_eq( + f.acceptor_handler->cur_stream_engine->count_server_handshake_called, + 1); } END_TEST @@ -1843,27 +1915,27 @@ TlsServiceServerTestFixture f; // First op causes accept handler to restart server handshake call. // The EOF will now handled in server handshake handler. - f.server_engine->next_result = gu::AsioStreamEngine::want_write; + f.acceptor_handler->next_stream_engine->next_result = gu::AsioStreamEngine::want_write; f.complete_server_handshake(); - f.server_engine->next_result = gu::AsioStreamEngine::eof; + ck_assert(f.acceptor_handler->cur_stream_engine != nullptr); + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::eof; f.server_io_service.run_one(); - // Acceptor silently discards accepted socket which fails during - // handshake and restarts async accept. - ck_assert(f.acceptor_handler->accepted_socket() == 0); - ck_assert(f.server_engine->count_server_handshake_called == 2); + ck_assert_int_eq( + f.acceptor_handler->cur_stream_engine->count_server_handshake_called, + 2); } END_TEST START_TEST(test_server_handshake_error) { TlsServiceServerTestFixture f; - f.server_engine->next_result = gu::AsioStreamEngine::error; - f.server_engine->next_error = EPIPE; + f.acceptor_handler->next_stream_engine->next_result + = gu::AsioStreamEngine::error; + f.acceptor_handler->next_stream_engine->next_error = EPIPE; f.complete_server_handshake(); - // Acceptor silently discards accepted socket which fails during - // handshake and restarts async accept. - ck_assert(f.acceptor_handler->accepted_socket() == 0); - ck_assert(f.server_engine->count_server_handshake_called == 1); + ck_assert_int_eq( + f.acceptor_handler->cur_stream_engine->count_server_handshake_called, + 1); } END_TEST @@ -1872,15 +1944,35 @@ TlsServiceServerTestFixture f; // First op causes accept handler to restart server handshake call. // The error will now handled in server handshake handler. - f.server_engine->next_result = gu::AsioStreamEngine::want_write; + f.acceptor_handler->next_stream_engine->next_result = gu::AsioStreamEngine::want_write; f.complete_server_handshake(); - f.server_engine->next_result = gu::AsioStreamEngine::error; - f.server_engine->next_error = EPIPE; + ck_assert(f.acceptor_handler->cur_stream_engine != nullptr); + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::error; + f.acceptor_handler->cur_stream_engine->next_error = EPIPE; f.server_io_service.run_one(); - // Acceptor silently discards accepted socket which fails during - // handshake and restarts async accept. - ck_assert(f.acceptor_handler->accepted_socket() == 0); - ck_assert(f.server_engine->count_server_handshake_called == 2); + ck_assert_int_eq(f.acceptor_handler->cur_stream_engine->count_server_handshake_called, 2); +} +END_TEST + +START_TEST(test_accept_after_server_handshake_error) +{ + TlsServiceServerTestFixture f; + f.acceptor_handler->next_stream_engine->next_result + = gu::AsioStreamEngine::error; + f.acceptor_handler->next_stream_engine->next_error = EPIPE; + f.complete_server_handshake(); + ck_assert(f.acceptor_handler->cur_stream_engine != nullptr); + ck_assert_int_eq( + f.acceptor_handler->cur_stream_engine->count_server_handshake_called, + 1); + + f.acceptor_handler->cur_stream_engine->next_error = 0; + f.run_async_connect(); + f.complete_server_handshake(); + ck_assert(f.acceptor_handler->accepted_socket() != nullptr); + ck_assert_int_eq( + f.acceptor_handler->cur_stream_engine->count_server_handshake_called, + 1); } END_TEST @@ -1895,25 +1987,25 @@ cbs[1] = gu::AsioConstBuffer(); f.socket->async_write(cbs, f.socket_handler); f.client_io_service.run_one(); - f.server_engine->next_result = gu::AsioStreamEngine::want_read; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::want_read; std::array buf; f.acceptor_handler->accepted_socket()->async_read( gu::AsioMutableBuffer(buf.data(), buf.size()), f.acceptor_handler->accepted_handler()); - f.run_server_while([&f]() { return f.server_engine->count_read_called < 1; }); - ck_assert(f.server_engine->count_read_called == 1); + f.run_server_while([&f]() { return f.acceptor_handler->cur_stream_engine->count_read_called < 1; }); + ck_assert(f.acceptor_handler->cur_stream_engine->count_read_called == 1); ck_assert(f.acceptor_handler->accepted_handler()->bytes_read() == 4); // Write socket to make accepted socket readable, but do not start // async read to simulate stream engine internal operation. f.socket->async_write(cbs, f.socket_handler); f.client_io_service.reset(); f.client_io_service.run_one(); - f.server_engine->next_result = gu::AsioStreamEngine::success; - const auto expect_count_read_called = f.server_engine->count_read_called + 1; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::success; + const size_t expect_count_read_called = f.acceptor_handler->cur_stream_engine->count_read_called + 1; f.run_server_while( - [&f, expect_count_read_called]() { return f.server_engine->count_read_called < expect_count_read_called; }); - ck_assert(f.server_engine->count_read_called == expect_count_read_called); + [&f, expect_count_read_called]() { return f.acceptor_handler->cur_stream_engine->count_read_called < expect_count_read_called; }); + ck_assert(f.acceptor_handler->cur_stream_engine->count_read_called == expect_count_read_called); // Extra read should just call read() but the communication should // be internal, the handler should not see received data. ck_assert(f.acceptor_handler->accepted_handler()->bytes_read() == 4); @@ -1931,23 +2023,23 @@ cbs[1] = gu::AsioConstBuffer(); f.socket->async_write(cbs, f.socket_handler); f.client_io_service.run_one(); - f.server_engine->next_result = gu::AsioStreamEngine::want_write; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::want_write; std::array buf; f.acceptor_handler->accepted_socket()->async_read( gu::AsioMutableBuffer(buf.data(), buf.size()), f.acceptor_handler->accepted_handler()); - const auto expect_count_read_called = f.server_engine->count_read_called + 1; + const size_t expect_count_read_called = f.acceptor_handler->cur_stream_engine->count_read_called + 1; f.run_server_while( - [&f, expect_count_read_called]() { return f.server_engine->count_read_called < expect_count_read_called; }); + [&f, expect_count_read_called]() { return f.acceptor_handler->cur_stream_engine->count_read_called < expect_count_read_called; }); - ck_assert(f.server_engine->count_read_called == expect_count_read_called); + ck_assert(f.acceptor_handler->cur_stream_engine->count_read_called == expect_count_read_called); ck_assert(f.acceptor_handler->accepted_handler()->bytes_read() == 4); f.run_server_while( - [&f, expect_count_read_called]() { return f.server_engine->count_read_called < expect_count_read_called + 1; }); + [&f, expect_count_read_called]() { return f.acceptor_handler->cur_stream_engine->count_read_called < expect_count_read_called + 1; }); // The result want_write means that the previous operation // (in this case read) must be called once again once the // socket becomes writable. - ck_assert(f.server_engine->count_read_called == expect_count_read_called + 1); + ck_assert(f.acceptor_handler->cur_stream_engine->count_read_called == expect_count_read_called + 1); } END_TEST @@ -1962,8 +2054,8 @@ gu::AsioMutableBuffer(buf.data(), buf.size()), f.acceptor_handler->accepted_handler()); f.run_server_while( - [&f]() { return f.server_engine->count_read_called < 1; }); - ck_assert(f.server_engine->count_read_called == 1); + [&f]() { return f.acceptor_handler->cur_stream_engine->count_read_called < 1; }); + ck_assert(f.acceptor_handler->cur_stream_engine->count_read_called == 1); ck_assert( f.acceptor_handler->accepted_handler()->last_error_code().is_eof()); } @@ -1977,15 +2069,15 @@ // Socket close makes the socket readable, but we override // the return value with error. f.socket->close(); - f.server_engine->next_result = gu::AsioStreamEngine::error; - f.server_engine->next_error = EPIPE; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::error; + f.acceptor_handler->cur_stream_engine->next_error = EPIPE; std::array buf; f.acceptor_handler->accepted_socket()->async_read( gu::AsioMutableBuffer(buf.data(), buf.size()), f.acceptor_handler->accepted_handler()); f.run_server_while( - [&f]() { return f.server_engine->count_read_called < 1; }); - ck_assert(f.server_engine->count_read_called == 1); + [&f]() { return f.acceptor_handler->cur_stream_engine->count_read_called < 1; }); + ck_assert(f.acceptor_handler->cur_stream_engine->count_read_called == 1); ck_assert(f.acceptor_handler->accepted_handler()->last_error_code().value() == EPIPE); } @@ -1997,7 +2089,7 @@ f.complete_server_handshake(); ck_assert(f.acceptor_handler->accepted_socket() != 0); - f.server_engine->next_result = gu::AsioStreamEngine::want_read; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::want_read; std::array cbs; cbs[0] = gu::AsioConstBuffer("writ", 4); cbs[1] = gu::AsioConstBuffer(); @@ -2005,7 +2097,7 @@ cbs, f.acceptor_handler->accepted_handler()); f.server_io_service.run_one(); ck_assert(f.acceptor_handler->accepted_handler()->bytes_written() == 4); - ck_assert(f.server_engine->count_write_called == 1); + ck_assert(f.acceptor_handler->cur_stream_engine->count_write_called == 1); // Write to client socket to make server side socket readable f.socket->async_write(cbs, f.socket_handler); f.client_io_service.reset(); @@ -2014,9 +2106,9 @@ // Now the server side socket should become readable and // the second call to write should happen. f.run_server_while( - [&]() { return f.server_engine->count_write_called < 2; }); + [&]() { return f.acceptor_handler->cur_stream_engine->count_write_called < 2; }); ck_assert(f.acceptor_handler->accepted_handler()->bytes_written() == 4); - ck_assert(f.server_engine->count_write_called == 2); + ck_assert(f.acceptor_handler->cur_stream_engine->count_write_called == 2); } END_TEST @@ -2026,7 +2118,7 @@ f.complete_server_handshake(); ck_assert(f.acceptor_handler->accepted_socket() != 0); - f.server_engine->next_result = gu::AsioStreamEngine::want_write; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::want_write; std::array cbs; cbs[0] = gu::AsioConstBuffer("writ", 4); cbs[1] = gu::AsioConstBuffer(); @@ -2034,13 +2126,13 @@ cbs, f.acceptor_handler->accepted_handler()); f.server_io_service.run_one(); ck_assert(f.acceptor_handler->accepted_handler()->bytes_written() == 4); - ck_assert(f.server_engine->count_write_called == 1); + ck_assert(f.acceptor_handler->cur_stream_engine->count_write_called == 1); // Now the server side socket should remain writable and the // the second call to write should happen. f.run_server_while( - [&f]() { return f.server_engine->count_write_called < 2; }); + [&f]() { return f.acceptor_handler->cur_stream_engine->count_write_called < 2; }); ck_assert(f.acceptor_handler->accepted_handler()->bytes_written() == 4); - ck_assert(f.server_engine->count_write_called == 2); + ck_assert(f.acceptor_handler->cur_stream_engine->count_write_called == 2); } END_TEST @@ -2050,7 +2142,7 @@ f.complete_server_handshake(); ck_assert(f.acceptor_handler->accepted_socket() != 0); - f.server_engine->next_result = gu::AsioStreamEngine::want_read; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::want_read; std::array cbs; cbs[0] = gu::AsioConstBuffer("writ", 4); cbs[1] = gu::AsioConstBuffer(); @@ -2058,16 +2150,16 @@ cbs, f.acceptor_handler->accepted_handler()); f.server_io_service.run_one(); ck_assert(f.acceptor_handler->accepted_handler()->bytes_written() == 4); - ck_assert(f.server_engine->count_write_called == 1); + ck_assert(f.acceptor_handler->cur_stream_engine->count_write_called == 1); // Write to client socket to make server side socket readable f.socket->async_write(cbs, f.socket_handler); f.client_io_service.reset(); f.client_io_service.run_one(); ck_assert(f.socket_handler->bytes_written() == 4); - f.server_engine->next_result = gu::AsioStreamEngine::eof; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::eof; f.run_server_while( - [&f] { return f.server_engine->count_write_called < 2; }); - ck_assert(f.server_engine->count_write_called == 2); + [&f] { return f.acceptor_handler->cur_stream_engine->count_write_called < 2; }); + ck_assert(f.acceptor_handler->cur_stream_engine->count_write_called == 2); ck_assert( f.acceptor_handler->accepted_handler()->last_error_code().is_eof()); } @@ -2079,16 +2171,16 @@ f.complete_server_handshake(); ck_assert(f.acceptor_handler->accepted_socket() != 0); - f.server_engine->next_result = gu::AsioStreamEngine::error; - f.server_engine->next_error = EPIPE; + f.acceptor_handler->cur_stream_engine->next_result = gu::AsioStreamEngine::error; + f.acceptor_handler->cur_stream_engine->next_error = EPIPE; std::array cbs; cbs[0] = gu::AsioConstBuffer("writ", 4); cbs[1] = gu::AsioConstBuffer(); f.acceptor_handler->accepted_socket()->async_write( cbs, f.acceptor_handler->accepted_handler()); f.run_server_while( - [&f] { return f.server_engine->count_write_called < 1; }); - ck_assert(f.server_engine->count_write_called == 1); + [&f] { return f.acceptor_handler->cur_stream_engine->count_write_called < 1; }); + ck_assert(f.acceptor_handler->cur_stream_engine->count_write_called == 1); // Write will succeed before the error is injected, so there will be // some bytes written. ck_assert(f.acceptor_handler->accepted_handler()->bytes_written() == 4); @@ -2586,6 +2678,10 @@ tcase_add_test(tc, test_server_handshake_error2); suite_add_tcase(s, tc); + tc = tcase_create("test_accept_after_server_handshake_error"); + tcase_add_test(tc, test_accept_after_server_handshake_error); + suite_add_tcase(s, tc); + tc = tcase_create("test_read_want_read"); tcase_add_test(tc, test_read_want_read); suite_add_tcase(s, tc); diff -Nru galera-4-26.4.18/garb/garb_main.cpp galera-4-26.4.20/garb/garb_main.cpp --- galera-4-26.4.18/garb/garb_main.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/garb/garb_main.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -20,12 +20,12 @@ { if (chdir("/")) // detach from potentially removable block devices { - gu_throw_error(errno) << "chdir(" << workdir << ") failed"; + gu_throw_system_error(errno) << "chdir(" << workdir << ") failed"; } if (!workdir.empty() && chdir(workdir.c_str())) { - gu_throw_error(errno) << "chdir(" << workdir << ") failed"; + gu_throw_system_error(errno) << "chdir(" << workdir << ") failed"; } if (pid_t pid = fork()) @@ -39,7 +39,7 @@ // I guess we want this to go to stderr as well; std::cerr << "Failed to fork daemon process: " << errno << " (" << strerror(errno) << ")"; - gu_throw_error(errno) << "Failed to fork daemon process"; + gu_throw_system_error(errno) << "Failed to fork daemon process"; } } @@ -47,7 +47,7 @@ if (setsid()<0) // become a new process leader, detach from terminal { - gu_throw_error(errno) << "setsid() failed"; + gu_throw_system_error(errno) << "setsid() failed"; } // umask(0); @@ -62,7 +62,7 @@ } else { - gu_throw_error(errno) << "Second fork failed"; + gu_throw_system_error(errno) << "Second fork failed"; } } @@ -77,7 +77,8 @@ { if (open("/dev/null", O_RDONLY) < 0) { - gu_throw_error(errno) << "Unable to open /dev/null for fd " << fd; + gu_throw_system_error(errno) + << "Unable to open /dev/null for fd " << fd; } } @@ -109,8 +110,9 @@ if (sigaction (SIGPIPE, &isa, NULL)) { - gu_throw_error(errno) << "Falied to install signal handler for signal " - << "SIGPIPE"; + gu_throw_system_error(errno) + << "Falied to install signal handler for signal " + << "SIGPIPE"; } RecvLoop loop (config); diff -Nru galera-4-26.4.18/garb/garb_recv_loop.cpp galera-4-26.4.20/garb/garb_recv_loop.cpp --- galera-4-26.4.18/garb/garb_recv_loop.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/garb/garb_recv_loop.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -49,14 +49,16 @@ if (sigaction (SIGTERM, &sa, NULL)) { - gu_throw_error(errno) << "Falied to install signal handler for signal " - << "SIGTERM"; + gu_throw_system_error(errno) + << "Falied to install signal handler for signal " + << "SIGTERM"; } if (sigaction (SIGINT, &sa, NULL)) { - gu_throw_error(errno) << "Falied to install signal handler for signal " - << "SIGINT"; + gu_throw_system_error(errno) + << "Failed to install signal handler for signal " + << "SIGINT"; } loop(); diff -Nru galera-4-26.4.18/gcache/src/gcache_page_store.cpp galera-4-26.4.20/gcache/src/gcache_page_store.cpp --- galera-4-26.4.18/gcache/src/gcache_page_store.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcache/src/gcache_page_store.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -104,7 +104,8 @@ if (0 != err) { delete_thr_ = pthread_t(-1); - gu_throw_error(err) << "Failed to create page file deletion thread"; + gu_throw_system_error(err) + << "Failed to create page file deletion thread"; } return true; @@ -164,8 +165,8 @@ if (0 != err) { - gu_throw_error(err) << "Failed to initialize page file deletion " - << "thread attributes"; + gu_throw_system_error(err) << "Failed to initialize page file deletion " + << "thread attributes"; } #ifdef GCACHE_DETACH_THREAD @@ -174,8 +175,8 @@ if (0 != err) { pthread_attr_destroy (&delete_page_attr_); - gu_throw_error(err) << "Failed to set DETACHED attribute to " - << "page file deletion thread"; + gu_throw_system_error(err) << "Failed to set DETACHED attribute to " + << "page file deletion thread"; } #endif /* GCACHE_DETACH_THREAD */ } diff -Nru galera-4-26.4.18/gcomm/src/asio_tcp.cpp galera-4-26.4.20/gcomm/src/asio_tcp.cpp --- galera-4-26.4.18/gcomm/src/asio_tcp.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcomm/src/asio_tcp.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2020 Codership Oy + * Copyright (C) 2012-2024 Codership Oy */ #include "asio_tcp.hpp" @@ -23,7 +23,7 @@ assert(ssize_t(recv_buf_size) >= 0); socket->set_receive_buffer_size(recv_buf_size); - auto cur_value(socket->get_receive_buffer_size()); + size_t cur_value(socket->get_receive_buffer_size()); log_debug << "socket recv buf size " << cur_value; if (cur_value < recv_buf_size && not asio_recv_buf_warned) { @@ -48,7 +48,7 @@ assert(ssize_t(send_buf_size) >= 0); socket->set_send_buffer_size(send_buf_size); - auto cur_value(socket->get_send_buffer_size()); + size_t cur_value(socket->get_send_buffer_size()); log_debug << "socket send buf size " << cur_value; if (cur_value < send_buf_size && not asio_send_buf_warned) { @@ -142,6 +142,7 @@ { if (ec) { + log_info << "Failed to establish connection: " << ec; FAILED_HANDLER(ec); return; } @@ -688,7 +689,7 @@ Acceptor (uri), net_ (net), acceptor_ (net_.io_service_.make_acceptor(uri)), - accepted_socket_() + next_socket_() { } gcomm::AsioTcpAcceptor::~AsioTcpAcceptor() @@ -704,13 +705,14 @@ { if (!error) { - auto socket(std::make_shared(net_, uri_, accepted_socket)); - socket->state_ = Socket::S_CONNECTED; - accepted_socket_ = socket; - log_debug << "accepted socket " << socket->id(); + next_socket_->socket_ = accepted_socket; + /* Notify upper layer which then calls accept() to acquire ownership. */ net_.dispatch(id(), Datagram(), ProtoUpMeta(error.value())); - acceptor_->async_accept(shared_from_this()); + assert(not next_socket_); } + acceptor_->async_accept( + shared_from_this(), + next_socket_ = std::make_shared(net_, uri_, nullptr)); } void gcomm::AsioTcpAcceptor::set_buf_sizes() @@ -725,7 +727,8 @@ acceptor_->open(uri); set_buf_sizes(); // Must be done before listen acceptor_->listen(uri); - acceptor_->async_accept(shared_from_this()); + next_socket_ = std::make_shared(net_, uri_, nullptr); + acceptor_->async_accept(shared_from_this(), next_socket_); } std::string gcomm::AsioTcpAcceptor::listen_addr() const @@ -741,9 +744,11 @@ gcomm::SocketPtr gcomm::AsioTcpAcceptor::accept() { - if (accepted_socket_->state() == Socket::S_CONNECTED) - { - accepted_socket_->async_receive(); - } - return accepted_socket_; + /* Note that the socket is not flagged as connected yet, it may + * still be in the middle of the handshake, e.g. if TLS is used. + * Once the handshake is complete, connect_handler() will be called + * which will notify the upper layer that the socket is ready. */ + auto ret = next_socket_; + next_socket_ = nullptr; + return ret; } diff -Nru galera-4-26.4.18/gcomm/src/asio_tcp.hpp galera-4-26.4.20/gcomm/src/asio_tcp.hpp --- galera-4-26.4.18/gcomm/src/asio_tcp.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcomm/src/asio_tcp.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2019 Codership Oy + * Copyright (C) 2010-2024 Codership Oy */ #ifndef GCOMM_ASIO_TCP_HPP @@ -131,7 +131,7 @@ AsioProtonet& net_; std::shared_ptr acceptor_; - SocketPtr accepted_socket_; + std::shared_ptr next_socket_; }; #include "gu_enable_non_virtual_dtor.hpp" diff -Nru galera-4-26.4.18/gcomm/src/evs_proto.cpp galera-4-26.4.20/gcomm/src/evs_proto.cpp --- galera-4-26.4.18/gcomm/src/evs_proto.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcomm/src/evs_proto.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -674,7 +674,7 @@ void gcomm::evs::Proto::handle_install_timer() { gcomm_assert(state() == S_GATHER || state() == S_INSTALL); - log_warn << self_string() << " install timer expired"; + log_info << self_string() << " install timer expired"; bool is_cons(consensus_.is_consensus()); bool is_repr(is_representative(uuid())); @@ -2302,7 +2302,7 @@ // a join message from joining node. This is to reduce the probability // of install timeouts because of already ongoing cluster configuration // changes. - const auto is_join_message_with_self + const bool is_join_message_with_self = msg.type() == Message::EVS_T_JOIN && msg.node_list().find(my_uuid_) != msg.node_list().end(); if (state() == S_JOINING && not is_join_message_with_self) @@ -2621,7 +2621,6 @@ else if (state() != S_OPERATIONAL) { - log_warn << "user message in state " << to_string(state()); return ENOTCONN; } diff -Nru galera-4-26.4.18/gcomm/src/gcomm/protolay.hpp galera-4-26.4.20/gcomm/src/gcomm/protolay.hpp --- galera-4-26.4.18/gcomm/src/gcomm/protolay.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcomm/src/gcomm/protolay.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -287,7 +287,6 @@ { if (down_context_.empty() == true) { - log_warn << this << " down context(s) not set"; return ENOTCONN; } diff -Nru galera-4-26.4.18/gcomm/src/gmcast.cpp galera-4-26.4.20/gcomm/src/gmcast.cpp --- galera-4-26.4.18/gcomm/src/gmcast.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcomm/src/gmcast.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -715,9 +715,9 @@ if (AddrList::value(i).retry_cnt() > AddrList::value(i).max_retries()) { - log_warn << "discarding established (time wait) " - << est->remote_uuid() - << " (" << est->remote_addr() << ") "; + log_info << "discarding connection " << est->remote_uuid() << " (" + << est->remote_addr() << ") " + << "after " << AddrList::value(i).retry_cnt() << " retries"; erase_proto(proto_map_->find(est->socket()->id())); update_addresses(); return; diff -Nru galera-4-26.4.18/gcomm/src/gmcast_proto.cpp galera-4-26.4.20/gcomm/src/gmcast_proto.cpp --- galera-4-26.4.18/gcomm/src/gmcast_proto.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcomm/src/gmcast_proto.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -282,9 +282,9 @@ void gcomm::gmcast::Proto::handle_failed(const Message& hs) { - log_warn << "handshake with " << remote_uuid_ << " " - << remote_addr_ << " failed: '" - << hs.error() << "'"; + log_debug << "handshake with " << remote_uuid_ << " " + << remote_addr_ << " failed: '" + << hs.error() << "'"; set_state(S_FAILED); if (hs.error() == gmcast_proto_err_evicted) { @@ -300,7 +300,7 @@ { if (gmcast_.prim_view_reached()) { - log_warn << "Received duplicate UUID error from other node " + log_info << "Received duplicate UUID error from other node " << "while in primary component. This may mean that " << "this node's IP address has changed. Will close " << "connection and keep on retrying"; diff -Nru galera-4-26.4.18/gcomm/src/pc_proto.cpp galera-4-26.4.20/gcomm/src/pc_proto.cpp --- galera-4-26.4.18/gcomm/src/pc_proto.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcomm/src/pc_proto.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -183,6 +183,21 @@ } } +static std::string send_error_str(int const err) +{ + std::ostringstream os; + switch (err) + { + case 0: os << "Success"; break; + case EAGAIN: + os << "Cluster configuration change in progress or flow control active"; + break; + case ENOTCONN: os << "Not connected to the cluster"; break; + default: os << "Unknown error: " << err; break; + } + return os.str(); +} + int gcomm::pc::Proto::send_install(bool bootstrap, int weight) { gcomm_assert(bootstrap == false || weight == -1); @@ -227,10 +242,10 @@ serialize(pci, buf); Datagram dg(buf); int ret = send_down(dg, ProtoDownMeta()); - if (ret != 0) + if (ret) { - log_warn << self_id() << " sending install message failed: " - << strerror(ret); + log_info << "sending install message for new primary component failed: " + << send_error_str(ret) << ", will retry in next configuration"; } return ret; } @@ -579,14 +594,14 @@ if (closing_ == false && ignore_sb_ == true && have_split_brain(view)) { // configured to ignore split brain - log_warn << "Ignoring possible split-brain " + log_info << "Ignoring possible split-brain " << "(allowed by configuration) from view:\n" << current_view_ << "\nto view:\n" << view; } else if (closing_ == false && ignore_quorum_ == true) { // configured to ignore lack of quorum - log_warn << "Ignoring lack of quorum " + log_info << "Ignoring lack of quorum " << "(allowed by configuration) from view:\n" << current_view_ << "\nto view:\n" << view; } @@ -964,7 +979,8 @@ if (last_prim_uuids.empty() == true) { - log_warn << "no nodes coming from prim view, prim not possible"; + log_info << "No nodes coming from primary view, " + << "primary view is not possible"; return false; } @@ -1641,7 +1657,9 @@ } else if (ret != EAGAIN) { - log_warn << "Proto::handle_down: " << strerror(ret); + log_warn << "Got unexpected error code from send in " + "pc::Proto::handle_down(): " + << ret; } pop_header(um, dg); diff -Nru galera-4-26.4.18/gcs/src/CMakeLists.txt galera-4-26.4.20/gcs/src/CMakeLists.txt --- galera-4-26.4.18/gcs/src/CMakeLists.txt 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/CMakeLists.txt 2024-07-30 05:28:41.000000000 +0000 @@ -22,6 +22,7 @@ gcs_fc.cpp gcs.cpp gcs_gcomm.cpp + gcs_error.cpp ) # diff -Nru galera-4-26.4.18/gcs/src/SConscript galera-4-26.4.20/gcs/src/SConscript --- galera-4-26.4.18/gcs/src/SConscript 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/SConscript 2024-07-30 05:28:41.000000000 +0000 @@ -51,6 +51,7 @@ gcs_fc.cpp gcs.cpp gcs_gcomm.cpp + gcs_error.cpp ''') #libgcs_env.VariantDir('.gcs', '.', duplicate=0) libgcs_env.StaticLibrary('gcs', libgcs_sources) diff -Nru galera-4-26.4.18/gcs/src/gcs.cpp galera-4-26.4.20/gcs/src/gcs.cpp --- galera-4-26.4.18/gcs/src/gcs.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -16,6 +16,7 @@ #include "gcs_fifo_lite.hpp" #include "gcs_sm.hpp" #include "gcs_gcache.hpp" +#include "gcs_error.hpp" #include #include @@ -29,6 +30,8 @@ #include #include +#include + const char* gcs_node_state_to_str (gcs_node_state_t state) { static const char* str[GCS_NODE_STATE_MAX + 1] = @@ -424,7 +427,7 @@ case -ENOTCONN: case -ECONNABORTED: if (NULL != warning) { - gu_warn ("%s: %d (%s)", warning, err, strerror(-err)); + gu_info ("%s: %d (%s)", warning, err, gcs_error_str(-err)); } err = 0; break; @@ -454,7 +457,7 @@ !(err = gu_mutex_lock (&conn->fc_lock))); if (gu_unlikely(err)) { - gu_fatal ("Mutex lock failed: %d (%s)", err, strerror(err)); + gu_fatal ("Mutex lock failed: %ld (%s)", err, strerror(err)); abort(); } @@ -489,7 +492,8 @@ conn->stop_sent_dec(1); } - gu_debug ("SENDING FC_STOP (local seqno: %lld, fc_offset: %ld): %d", + gu_debug("SENDING FC_STOP (local seqno: %" PRId64 + ", fc_offset: %ld): %d", conn->local_act_id, conn->fc_offset, ret); } else @@ -519,7 +523,7 @@ !(err = gu_mutex_lock (&conn->fc_lock))); if (gu_unlikely(err)) { - gu_fatal ("Mutex lock failed: %d (%s)", err, strerror(err)); + gu_fatal ("Mutex lock failed: %ld (%s)", err, strerror(err)); abort(); } @@ -551,7 +555,8 @@ conn->stop_sent_inc(1); } - gu_debug ("SENDING FC_CONT (local seqno: %lld, fc_offset: %ld): %d", + gu_debug("SENDING FC_CONT (local seqno: %" PRId64 + ", fc_offset: %ld): %d", conn->local_act_id, conn->fc_offset, ret); } else @@ -659,16 +664,17 @@ if (!allowed[new_state][old_state]) { if (old_state != new_state) { - gu_warn ("GCS: Shifting %s -> %s is not allowed (TO: %lld)", - gcs_conn_state_str[old_state], - gcs_conn_state_str[new_state], conn->global_seqno); + gu_warn("GCS: Shifting %s -> %s is not allowed (TO: %" PRId64 ")", + gcs_conn_state_str[old_state], + gcs_conn_state_str[new_state], conn->global_seqno); } return false; } if (old_state != new_state) { - gu_info ("Shifting %s -> %s (TO: %lld)", gcs_conn_state_str[old_state], - gcs_conn_state_str[new_state], conn->global_seqno); + gu_info("Shifting %s -> %s (TO: %" PRId64 ")", + gcs_conn_state_str[old_state], gcs_conn_state_str[new_state], + conn->global_seqno); conn->state = new_state; } @@ -738,8 +744,8 @@ int ret; if ((ret = _release_flow_control (conn))) { - gu_fatal ("Failed to release flow control: %ld (%s)", - ret, strerror(ret)); + gu_fatal ("Failed to release flow control: %d (%s)", + ret, gcs_error_str(ret)); gcs_close (conn); abort(); } @@ -792,7 +798,7 @@ -EPROTO); if (err < 0 && !(err == -ENOTCONN || err == -EBADFD)) { gu_fatal ("Failed to send State Transfer Request rejection: " - "%zd (%s)", err, (strerror (-err))); + "%zd (%s)", err, (gcs_error_str (-err))); assert (0); return -ENOTRECOVERABLE; // failed to clear donor status, } @@ -847,8 +853,8 @@ if (GCS_CONN_JOINER == conn->state) { ret = _release_sst_flow_control (conn); if (ret < 0) { - gu_fatal ("Releasing SST flow control failed: %ld (%s)", - ret, strerror (-ret)); + gu_fatal ("Releasing SST flow control failed: %d (%s)", + ret, gcs_error_str (-ret)); abort(); } conn->timeout = GU_TIME_ETERNITY; @@ -863,7 +869,7 @@ gu_debug("Become joined, FC offset %ld", conn->fc_offset); /* One of the cases when the node can become SYNCED */ if ((ret = gcs_send_sync (conn))) { - gu_warn ("Sending SYNC failed: %ld (%s)", ret, strerror (-ret)); + gu_warn ("Sending SYNC failed: %d (%s)", ret, gcs_error_str(-ret)); } } else { @@ -956,11 +962,12 @@ switch (err) { case -ENOTCONN: - gu_warn ("Sending JOIN failed: %d (%s). " - "Will retry in new primary component.", err,strerror(-err)); + gu_info("Sending JOIN failed: %s. " + "Will retry in new primary component.", + gcs_error_str(-err)); return 0; default: - gu_error ("Sending JOIN failed: %d (%s).", err, strerror(-err)); + gu_error("Sending JOIN failed: %d (%s).", err, gcs_error_str(-err)); return err; } } @@ -1103,7 +1110,7 @@ } if (old_state != conn->state) { - gu_info ("Restored state %s -> %s (%lld)", + gu_info ("Restored state %s -> %s (%" PRId64 ")", gcs_conn_state_str[old_state], gcs_conn_state_str[conn->state], conn->global_seqno); } @@ -1112,7 +1119,7 @@ case GCS_CONN_JOINED: /* One of the cases when the node can become SYNCED */ if ((ret = gcs_send_sync(conn)) < 0) { - gu_warn ("CC: sending SYNC failed: %ld (%s)", ret, strerror (-ret)); + gu_warn ("CC: sending SYNC failed: %ld (%s)", ret, gcs_error_str (-ret)); } break; case GCS_CONN_JOINER: @@ -1134,7 +1141,7 @@ { if ((gcs_seqno_t)conn->my_idx == rcvd.id) { int const donor_idx = (int)rcvd.id; // to pacify valgrind - gu_debug("Got GCS_ACT_STATE_REQ to %i, my idx: %ld", + gu_debug("Got GCS_ACT_STATE_REQ to %i, my idx: %d", donor_idx, conn->my_idx); // rewrite to pass global seqno for application rcvd.id = conn->global_seqno; @@ -1153,7 +1160,7 @@ gcs_handle_state_change (gcs_conn_t* conn, const struct gcs_act* act) { - gu_debug ("Got '%s' dated %lld", gcs_act_type_to_str (act->type), + gu_debug ("Got '%s' dated %" PRId64, gcs_act_type_to_str (act->type), gcs_seqno_gtoh(*(gcs_seqno_t*)act->buf)); void* buf = malloc (act->buf_len); @@ -1385,7 +1392,7 @@ /* if called from gcs_close(), we need to synchronize with gcs_recv_thread at this point */ if ((ret = gu_thread_join (conn->recv_thread, NULL))) { - gu_error ("Failed to join recv_thread(): %d (%s)", + gu_error ("Failed to join recv_thread(): %ld (%s)", -ret, strerror(-ret)); } else { @@ -1451,7 +1458,8 @@ if (gu_unlikely(ret <= 0)) { - gu_debug ("gcs_core_recv returned %d: %s", ret, strerror(-ret)); + gu_debug("gcs_core_recv returned %zd: %s", ret, + gcs_error_str(-ret)); if (-ETIMEDOUT == ret && _handle_timeout(conn)) continue; @@ -1490,8 +1498,15 @@ { ret = gcs_handle_actions (conn, rcvd); - if (gu_unlikely(ret < 0)) { // error - gu_debug ("gcs_handle_actions returned %d: %s", + if (gu_unlikely(ret <= 0 && GCS_ACT_COMMIT_CUT == rcvd.act.type)) + { + /* Commit cut will be discarded, the buffer needs to be + * freed */ + ::free(const_cast(rcvd.act.buf)); + } + if (gu_unlikely(ret < 0)) + { // error + gu_debug ("gcs_handle_actions returned %zd: %s", ret, strerror(-ret)); break; } @@ -1565,8 +1580,8 @@ } if (gu_unlikely(send_stop) && (ret = gcs_fc_stop_end(conn))) { - gu_error ("gcs_fc_stop() returned %d: %s", - ret, strerror(-ret)); + gu_error ("gcs_fc_stop() returned %zd: %s", + ret, gcs_error_str(-ret)); break; } } @@ -1594,7 +1609,7 @@ else if (conn->my_idx == rcvd.sender_idx) { gu_debug("Discarding: unordered local action not in repl_q: " - "{ {%p, %zd, %s}, %d, %lld }.", + "{ {%p, %zd, %s}, %d, %" PRId64 " }.", rcvd.act.buf, rcvd.act.buf_len, gcs_act_type_to_str(rcvd.act.type), rcvd.sender_idx, rcvd.id); @@ -1602,7 +1617,7 @@ else { gu_fatal ("Protocol violation: unordered remote action: " - "{ {%p, %zd, %s}, %d, %lld }", + "{ {%p, %zd, %s}, %d, % " PRId64 " }", rcvd.act.buf, rcvd.act.buf_len, gcs_act_type_to_str(rcvd.act.type), rcvd.sender_idx, rcvd.id); @@ -1621,7 +1636,7 @@ (void)_close(conn, false); gcs_shift_state (conn, GCS_CONN_CLOSED); } - gu_info ("RECV thread exiting %d: %s", ret, strerror(-ret)); + gu_info ("RECV thread exiting %zd: %s", ret, strerror(-ret)); return NULL; } @@ -1638,7 +1653,7 @@ if ((ret = gcs_sm_enter (conn->sm, &tmp_cond, false, true))) { - gu_error("Failed to enter send monitor: %d (%s)", ret, strerror(-ret)); + gu_error("Failed to enter send monitor: %ld (%s)", ret, strerror(-ret)); return ret; } @@ -1665,7 +1680,7 @@ gcs_core_close (conn->core); } else { - gu_error ("Failed to open channel '%s' at '%s': %d (%s)", + gu_error ("Failed to open channel '%s' at '%s': %ld (%s)", channel, url, ret, strerror(-ret)); } } @@ -1699,7 +1714,7 @@ /* _close() has already been called by gcs_recv_thread() and it is taking care of cleanup, just join the thread */ if ((ret = gu_thread_join (conn->recv_thread, NULL))) { - gu_error ("Failed to join recv_thread(): %d (%s)", + gu_error ("Failed to join recv_thread(): %ld (%s)", -ret, strerror(-ret)); } else { @@ -1744,7 +1759,7 @@ * to acquire the lock and give up gracefully */ } else { - gu_debug("gcs_destroy: gcs_sm_enter() err = %d", err); + gu_debug("gcs_destroy: gcs_sm_enter() err = %ld", err); // We should still cleanup resources } @@ -1754,12 +1769,12 @@ gcs_sm_destroy (conn->sm); if ((err = gcs_fifo_lite_destroy (conn->repl_q))) { - gu_debug ("Error destroying repl FIFO: %d (%s)", err, strerror(-err)); + gu_debug ("Error destroying repl FIFO: %ld (%s)", err, strerror(-err)); return err; } if ((err = gcs_core_destroy (conn->core))) { - gu_debug ("Error destroying core: %d (%s)", err, strerror(-err)); + gu_debug ("Error destroying core: %ld (%s)", err, strerror(-err)); return err; } @@ -1889,9 +1904,10 @@ if (ret < 0) { /* remove item from the queue, it will never be delivered */ - gu_warn ("Send action {%p, %zd, %s} returned %d (%s)", - act->buf, act->size,gcs_act_type_to_str(act->type), - ret, strerror(-ret)); + gu_debug( + "Send action {%p, %" PRId32 ", %s} returned %ld (%s)", + act->buf, act->size, gcs_act_type_to_str(act->type), + ret, gcs_error_str(-ret)); if (!gcs_fifo_lite_remove (conn->repl_q)) { gu_fatal ("Failed to remove unsent item from repl_q"); @@ -1941,7 +1957,7 @@ if (orig_buf != act->buf) // action was allocated in gcache { - gu_debug("Freeing gcache buffer %p after receiving %d", + gu_debug("Freeing gcache buffer %p after receiving %ld", act->buf, ret); gcs_gcache_free (conn->gcache, act->buf); act->buf = orig_buf; @@ -2126,19 +2142,19 @@ if (conn->queue_len > 0) { gu_warn ("Failed to send CONT message: %d (%s). " "Attempts left: %ld", - err, strerror(-err), conn->queue_len); + err, gcs_error_str(-err), conn->queue_len); } else { gu_fatal ("Last opportunity to send CONT message failed: " "%d (%s). Aborting to avoid cluster lock-up...", - err, strerror(-err)); + err, gcs_error_str(-err)); gcs_close(conn); gu_abort(); } } else if (gu_unlikely(send_sync) && (err = gcs_send_sync_end (conn))) { gu_warn ("Failed to send SYNC message: %d (%s). Will try later.", - err, strerror(-err)); + err, gcs_error_str(-err)); } return action->size; @@ -2260,11 +2276,19 @@ if (gcs_proto_ver(conn) < 1) { assert(code != 0); // should be here only our own initiative - log_error << "Not all group members support inconsistency voting. " - << "Reverting to old behavior: abort on error."; + log_info << "Not all group members support inconsistency voting. " + << "Reverting to old behavior: abort on error."; return 1; /* no voting with old protocol */ } + if (conn->state >= GCS_CONN_JOINER) + { + assert(code != 0); // should be here only our own initiative + log_info << "Can't vote when not at least JOINED. " + << "Assuming inconsistency. Full SST is required"; + return 1; /* Error applying IST event */ + } + int const err(gu_mutex_lock(&conn->vote_lock_)); if (gu_unlikely(0 != err)) { diff -Nru galera-4-26.4.18/gcs/src/gcs_act_proto.cpp galera-4-26.4.20/gcs/src/gcs_act_proto.cpp --- galera-4-26.4.18/gcs/src/gcs_act_proto.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_act_proto.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -85,7 +85,7 @@ frag->proto_ver = ((uint8_t*)buf)[PROTO_PV_OFFSET]; if (gu_unlikely(buf_len < PROTO_DATA_OFFSET)) { - gu_error ("Action message too short: %zu, expected at least %d", + gu_error ("Action message too short: %zu, expected at least %zu", buf_len, PROTO_DATA_OFFSET); return -EBADMSG; } diff -Nru galera-4-26.4.18/gcs/src/gcs_act_proto.hpp galera-4-26.4.20/gcs/src/gcs_act_proto.hpp --- galera-4-26.4.18/gcs/src/gcs_act_proto.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_act_proto.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -24,8 +24,10 @@ * 2 - support for commit cut in state exchange msg * 3 - fix for commit cut tracking issue * (needs protocol version bump to keep it identical on all nodes) + * 4 - fix for the error voting protocol + * (must keep it identical on all nodes) */ -#define GCS_PROTO_MAX 3 +#define GCS_PROTO_MAX 4 /*! Internal action fragment data representation */ typedef struct gcs_act_frag diff -Nru galera-4-26.4.18/gcs/src/gcs_core.cpp galera-4-26.4.20/gcs/src/gcs_core.cpp --- galera-4-26.4.18/gcs/src/gcs_core.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_core.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -15,6 +15,7 @@ #include "gcs_backend.hpp" #include "gcs_comp_msg.hpp" #include "gcs_code_msg.hpp" +#include "gcs_error.hpp" #include "gcs_fifo_lite.hpp" #include "gcs_group.hpp" #include "gcs_gcache.hpp" @@ -27,6 +28,8 @@ #include // for mempcpy #include +#include + using namespace gcs::core; bool @@ -216,14 +219,14 @@ core->state = CORE_NON_PRIMARY; } else { - gu_error ("Failed to open backend connection: %d (%s)", + gu_error ("Failed to open backend connection: %ld (%s)", ret, strerror(-ret)); core->backend.destroy (&core->backend); } } else { - gu_error ("Failed to initialize backend using '%s': %d (%s)", + gu_error ("Failed to initialize backend using '%s': %ld (%s)", url, ret, strerror(-ret)); } @@ -351,7 +354,7 @@ } else { ret = core_error (conn->state); - gu_error ("Failed to access core FIFO: %d (%s)", ret, strerror (-ret)); + gu_error ("Failed to access core FIFO: %zd (%s)", ret, strerror (-ret)); return ret; } @@ -486,7 +489,7 @@ /* sometimes - like in case of component message, we may need to * do reallocation 2 times. This should be fixed in backend */ void* msg = gu_realloc (recv_msg->buf, ret); - gu_debug ("Reallocating buffer from %d to %d bytes", + gu_debug ("Reallocating buffer from %d to %ld bytes", recv_msg->buf_len, ret); if (msg) { /* try again */ @@ -500,7 +503,7 @@ } else { /* realloc unsuccessfull, old recv_buf remains */ - gu_error ("Failed to reallocate buffer to %d bytes", ret); + gu_error ("Failed to reallocate buffer to %ld bytes", ret); ret = -ENOMEM; break; } @@ -509,7 +512,7 @@ assert(recv_msg->buf); if (gu_unlikely(ret < 0)) { - gu_debug ("returning %d: %s\n", ret, strerror(-ret)); + gu_debug ("returning %ld: %s\n", ret, strerror(-ret)); } return ret; @@ -593,8 +596,10 @@ /* NOTE! local_act cannot be used after this point */ /* sanity check */ if (gu_unlikely(sent_act_id != frg.act_id)) { - gu_fatal ("FIFO violation: expected sent_act_id %lld " - "found %lld", sent_act_id, frg.act_id); + gu_fatal("FIFO violation: expected sent_act_id %" PRId64 + " " + "found %" PRId64, + sent_act_id, frg.act_id); ret = -ENOTRECOVERABLE; } if (gu_unlikely(act->act.buf_len != ret)) { @@ -637,7 +642,7 @@ ret = gcs_group_handle_state_request (group, act); assert (ret <= 0 || ret == act->act.buf_len); #ifdef GCS_FOR_GARB - if (ret < 0) gu_fatal ("Handling state request failed: %d",ret); + if (ret < 0) gu_fatal ("Handling state request failed: %ld",ret); act->act.buf = NULL; } else { @@ -661,7 +666,7 @@ } else { /* Non-primary conf, foreign message - ignore */ - gu_warn ("Action message in non-primary configuration from " + gu_info ("Action message in non-primary configuration from " "member %d", msg->sender_idx); ret = 0; } @@ -792,7 +797,7 @@ assert (GCS_MSG_COMPONENT == msg->type); if (msg->size < (ssize_t)sizeof(gcs_comp_msg_t)) { - gu_error ("Malformed component message (size %zd < %zd). Ignoring", + gu_error ("Malformed component message (size %d < %zu). Ignoring", msg->size, sizeof(gcs_comp_msg_t)); return 0; } @@ -811,7 +816,7 @@ ret = gcs_group_act_conf (group, rcvd, &core->proto_ver); if (ret < 0) { - gu_fatal ("Failed create PRIM CONF action: %d (%s)", + gu_fatal ("Failed create PRIM CONF action: %zd (%s)", ret, strerror (-ret)); assert (0); ret = -ENOTRECOVERABLE; @@ -834,11 +839,25 @@ &uuid, sizeof(uuid), GCS_MSG_STATE_UUID); - if (ret < 0) { + if (ret < 0) + { // if send() failed, it means new configuration change // is on the way. Probably should ignore. - gu_warn ("Failed to send state UUID: %d (%s)", - ret, strerror (-ret)); + switch (-ret) + { + case EAGAIN: + gu_info("Temporary failure in sending state UUID, " + "will try again in next primary component"); + break; + case ENOTCONN: + gu_info("Failed to send state UUID: Connection to " + "cluster was closed"); + break; + default: + gu_warn("Failed to send state UUID: %zd (%s)", ret, + gcs_error_str(-ret)); + break; + } } else { gu_info ("STATE_EXCHANGE: sent state UUID: " @@ -863,7 +882,7 @@ assert(act->buf == NULL); assert(act->buf_len == 0); act->type = GCS_ACT_ERROR; - gu_debug("comp msg error in core %d", -ret); + gu_debug("comp msg error in core %ld", -ret); } } else { // regular non-prim @@ -873,7 +892,7 @@ if (GCS_GROUP_NON_PRIMARY == ret) { // no error in comp msg ret = gcs_group_act_conf (group, rcvd, &core->proto_ver); if (ret < 0) { - gu_fatal ("Failed create NON-PRIM CONF action: %d (%s)", + gu_fatal ("Failed create NON-PRIM CONF action: %ld (%s)", ret, strerror (-ret)); assert (0); ret = -ENOTRECOVERABLE; @@ -895,7 +914,7 @@ assert(0); // fall through default: - gu_fatal ("Failed to handle component message: %d (%s)!", + gu_fatal ("Failed to handle component message: %ld (%s)!", ret, strerror (-ret)); assert(0); } @@ -946,7 +965,7 @@ // This may happen if new configuraiton chage goes on. // What shall we do in this case? Is it unrecoverable? gu_error ("STATE EXCHANGE: failed for: " GU_UUID_FORMAT - ": %d (%s)", + ": %zd (%s)", GU_UUID_ARGS(state_uuid), ret, strerror(-ret)); } gcs_state_msg_destroy (state); @@ -962,7 +981,7 @@ break; default: assert(ret < 0); - gu_error ("Failed to handle state UUID: %d (%s)", + gu_error ("Failed to handle state UUID: %zd (%s)", ret, strerror (-ret)); } } @@ -1011,7 +1030,7 @@ ret = gcs_group_act_conf (group, rcvd, &core->proto_ver); if (ret < 0) { - gu_fatal ("Failed create CONF action: %d (%s)", + gu_fatal ("Failed create CONF action: %zd (%s)", ret, strerror (-ret)); assert (0); ret = -ENOTRECOVERABLE; @@ -1027,7 +1046,7 @@ break; default: assert (ret < 0); - gu_error ("Failed to handle state message: %d (%s)", + gu_error ("Failed to handle state message: %zd (%s)", ret, strerror (-ret)); } gu_mutex_unlock (&core->send_lock); @@ -1115,7 +1134,7 @@ } break; default: - gu_error ("Iternal error. Unexpected message type %s from %ld", + gu_error ("Iternal error. Unexpected message type %s from %d", gcs_msg_type_string[msg->type], msg->sender_idx); assert (0); ret = -EPROTO; @@ -1129,7 +1148,10 @@ } } else { - gu_warn ("%s message from member %ld in non-primary configuration. " + /* Messages which were sent just before cluster partitioning may + * be delivered in the following non-primary configuration. This + * is expected behavior, so info log level is enough. */ + gu_info ("%s message from member %d in non-primary configuration. " "Ignored.", gcs_msg_type_string[msg->type], msg->sender_idx); } @@ -1141,7 +1163,7 @@ { if (gu_unlikely(msg->size != sizeof(causal_act_t))) { - gu_error("invalid causal act len %ld, expected %ld", + gu_error("invalid causal act len %d, expected %zu", msg->size, sizeof(causal_act_t)); return -EPROTO; } diff -Nru galera-4-26.4.18/gcs/src/gcs_defrag.cpp galera-4-26.4.20/gcs/src/gcs_defrag.cpp --- galera-4-26.4.18/gcs/src/gcs_defrag.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_defrag.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -7,6 +7,7 @@ #include "gcs_defrag.hpp" #include +#include #include #include @@ -58,9 +59,9 @@ /* df->sent_id was aborted halfway and is being taken care of * by the sender thread. Forget about it. * Reinit counters and continue with the new action. */ - gu_debug ("Local action %lld, size %ld reset.", - frg->act_id, frg->act_size); - df->frag_no = 0; + gu_debug("Local action %" PRId64 ", size %ld reset.", + frg->act_id, frg->act_size); + df->frag_no = 0; df->received = 0; df->tail = df->head; df->reset = false; @@ -83,18 +84,20 @@ } else if (frg->act_id == df->sent_id && frg->frag_no < df->frag_no) { /* gh172: tolerate duplicate fragments in production. */ - gu_warn ("Duplicate fragment %lld:%ld, expected %lld:%ld. " - "Skipping.", - frg->act_id, frg->frag_no, df->sent_id, df->frag_no); + gu_warn("Duplicate fragment %" PRId64 ":%ld, expected %" PRId64 + ":%ld. " + "Skipping.", + frg->act_id, frg->frag_no, df->sent_id, df->frag_no); df->frag_no--; // revert counter in hope that we get good frag assert(0); return 0; } else { gu_error ("Unordered fragment received. Protocol error."); - gu_error ("Expected: %llu:%ld, received: %llu:%ld", - df->sent_id, df->frag_no, frg->act_id, frg->frag_no); - gu_error ("Contents: '%.*s'", frg->frag_len, (char*)frg->frag); + gu_error("Expected: %" PRId64 ":%ld, received: %" PRId64 ":%ld", + df->sent_id, df->frag_no, frg->act_id, frg->frag_no); + gu_error("Contents: '%.*s'", static_cast(frg->frag_len), + (char*)frg->frag); df->frag_no--; // revert counter in hope that we get good frag assert(0); return -EPROTO; @@ -122,18 +125,19 @@ if (!local && df->reset) { /* can happen after configuration change, just ignore this message calmly */ - gu_debug ("Ignoring fragment %lld:%ld (size %d) after reset", - frg->act_id, frg->frag_no, frg->act_size); + gu_debug("Ignoring fragment %" PRId64 + ":%ld (size %zu) after reset", + frg->act_id, frg->frag_no, frg->act_size); return 0; } else { ((char*)frg->frag)[frg->frag_len - 1] = '\0'; gu_error ("Unordered fragment received. Protocol error."); - gu_error ("Expected: any:0(first), received: %lld:%ld", - frg->act_id, frg->frag_no); - gu_error ("Contents: '%s', local: %s, reset: %s", - (char*)frg->frag, local ? "yes" : "no", - df->reset ? "yes" : "no"); + gu_error("Expected: any:0(first), received: %" PRId64 ":%lu", + frg->act_id, frg->frag_no); + gu_error("Contents: '%s', local: %s, reset: %s", + (char*)frg->frag, local ? "yes" : "no", + df->reset ? "yes" : "no"); assert(0); return -EPROTO; } diff -Nru galera-4-26.4.18/gcs/src/gcs_dummy.cpp galera-4-26.4.20/gcs/src/gcs_dummy.cpp --- galera-4-26.4.18/gcs/src/gcs_dummy.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_dummy.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -167,7 +167,7 @@ } else { ret = -EBADFD; // closing - gu_debug ("Returning %d: %s", ret, strerror(-ret)); + gu_debug ("Returning %ld: %s", ret, strerror(-ret)); } } else { @@ -189,7 +189,7 @@ const long max_pkt_size = backend->conn->max_pkt_size; if (pkt_size > max_pkt_size) { - gu_warn ("Requested packet size: %d, maximum possible packet size: %d", + gu_warn ("Requested packet size: %ld, maximum possible packet size: %ld", pkt_size, max_pkt_size); return (max_pkt_size - backend->conn->hdr_size); } @@ -230,7 +230,7 @@ } gcs_comp_msg_delete (comp); } - gu_debug ("Opened backend connection: %d (%s)", ret, strerror(-ret)); + gu_debug ("Opened backend connection: %ld (%s)", ret, strerror(-ret)); return ret; } diff -Nru galera-4-26.4.18/gcs/src/gcs_error.cpp galera-4-26.4.20/gcs/src/gcs_error.cpp --- galera-4-26.4.18/gcs/src/gcs_error.cpp 1970-01-01 00:00:00.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_error.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 Codership Oy + */ + +#include "gcs_error.hpp" + +#include +#include + +const char* gcs_error_str(int err) +{ + switch (err) + { + case EINTR: return "Operation interrupted"; + case EAGAIN: return "Operation failed temporarily"; + case EPERM: + case ENOTCONN: return "Not in primary component"; + case ECONNABORTED: return "Connection was closed"; + case EBADF: return "Connection not initialized"; + case ETIMEDOUT: return "Operation timed out"; + default: return strerror(err); + } +} + +const char* gcs_state_transfer_error_str(int err) +{ + switch (err) + { + case EAGAIN: + return "No donor candidates temporarily available in suitable state"; + case EHOSTUNREACH: return "Requested donor is not available"; + case EHOSTDOWN: return "Joiner and donor can't be the same node"; + default: return gcs_error_str(err); + } +} diff -Nru galera-4-26.4.18/gcs/src/gcs_error.hpp galera-4-26.4.20/gcs/src/gcs_error.hpp --- galera-4-26.4.18/gcs/src/gcs_error.hpp 1970-01-01 00:00:00.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_error.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2024 Codership Oy + */ + +/*! @file gcs_error.hpp + * + * Error code to error string translation according to GCS conventions. + */ + +#ifndef GCS_ERROR_HPP +#define GCS_ERROR_HPP + +/*! + * Return an error string associated with a system error code for gcs calls + * where the error code does not come from system call. As a fallback, + * error string for unhandled error codes are obtained by strerror() + * system call. + * + * This function follows the following conventions for system error + * codes for group communication errors: + * + * EAGAIN - Operation failed temporarily due to group configuration + * change or flow control. + * ENOTCONN, EPERM - Not in primary component. + * ECONNABORTED - Connection was closed while the operation was in progress. + * ETIMEDOUT - Operation timed out. + * EBADF - Connection was not initialized. + * + * @param err System error code. + * @return Error string describing the error condition. + */ +const char* gcs_error_str(int err); + +/*! + * Return and errorstring associated with a system error code for + * state transfer requests. As a fallback, error string for unhandled + * error codes are obtained by strerror() system call. + * + * The function follows the following conventions for system error codes + * for state transfer request errors (for details, see donor selection in + * gcs_group.cpp): + * + * EAGAIN - No donors available in suitable state. + * EHOSTUNREACH - Requested donor is not avaialble. + * EHOSTDOWN - Joiner and donor can't be the same node. + * + * @param err System error code. + * @return Error string describing state transfer error condition. + */ +const char* gcs_state_transfer_error_str(int err); + + +#endif /* GCS_ERROR_HPP */ diff -Nru galera-4-26.4.18/gcs/src/gcs_gcomm.cpp galera-4-26.4.20/gcs/src/gcs_gcomm.cpp --- galera-4-26.4.18/gcs/src/gcs_gcomm.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_gcomm.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -370,7 +370,7 @@ if ((err = gu_thread_create( &thd_, 0, run_fn, this)) != 0) { - gu_throw_error(err) << "Failed to create thread"; + gu_throw_system_error(err) << "Failed to create thread"; } thread_set_schedparam(thd_, schedparam_); @@ -585,7 +585,7 @@ i->second.segment()); if (ret < 0) { gu_throw_error(-ret) << "Failed to add member '" << uuid - << "' to component message."; + << "' to component message: " << -ret; } if (uuid == my_uuid) @@ -868,7 +868,7 @@ GCommConn::Ref ref(backend); if (ref.get() == 0) { - gu_throw_error(-EBADFD); + gu_throw_error(-EBADFD) << "Could not get status from gcomm backend"; } GCommConn& conn(*ref.get()); diff -Nru galera-4-26.4.18/gcs/src/gcs_group.cpp galera-4-26.4.20/gcs/src/gcs_group.cpp --- galera-4-26.4.18/gcs/src/gcs_group.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_group.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -8,6 +8,7 @@ #include "gcs_gcache.hpp" #include "gcs_priv.hpp" #include "gcs_code_msg.hpp" +#include "gcs_error.hpp" #include #include @@ -15,6 +16,7 @@ #include +#include #include std::string const GCS_VOTE_POLICY_KEY("gcs.vote_policy"); @@ -146,7 +148,7 @@ } } else { - gu_error ("Could not allocate %ld x %z bytes", nodes_num, + gu_error ("Could not allocate %ld x %zu bytes", nodes_num, sizeof(gcs_node_t)); } return ret; @@ -480,10 +482,10 @@ gu_info ("Quorum results:" "\n\tversion = %u," "\n\tcomponent = %s," - "\n\tconf_id = %lld," - "\n\tmembers = %d/%d (joined/total)," - "\n\tact_id = %lld," - "\n\tlast_appl. = %lld," + "\n\tconf_id = %" PRId64 "," + "\n\tmembers = %ld/%ld (joined/total)," + "\n\tact_id = %" PRId64 "," + "\n\tlast_appl. = %" PRId64 "," "\n\tprotocols = %d/%d/%d (gcs/repl/appl)," "\n\tvote policy= %d," "\n\tgroup UUID = " GU_UUID_FORMAT, @@ -543,7 +545,7 @@ new_nodes = group_nodes_init (group, comp); if (!new_nodes) { - gu_fatal ("Could not allocate memory for %ld-node component.", + gu_fatal ("Could not allocate memory for %d-node component.", gcs_comp_msg_num (comp)); assert(0); return (gcs_group_state_t)-ENOMEM; @@ -692,7 +694,7 @@ } else { gu_warn ("Stray state UUID msg: " GU_UUID_FORMAT - " from node %ld (%s), current group state %s", + " from node %d (%s), current group state %s", GU_UUID_ARGS((gu_uuid_t*)msg->buf), msg->sender_idx, group->nodes[msg->sender_idx].name, gcs_group_state_str[group->state]); @@ -726,7 +728,7 @@ } else { gu_debug ("STATE EXCHANGE: stray state msg: " GU_UUID_FORMAT - " from node %ld (%s), current state UUID: " + " from node %d (%s), current state UUID: " GU_UUID_FORMAT, GU_UUID_ARGS(state_uuid), msg->sender_idx, gcs_state_msg_name(state), @@ -737,7 +739,7 @@ } } else { - gu_warn ("Could not parse state message from node %d", + gu_warn ("Could not parse state message from node %d, %s", msg->sender_idx, group->nodes[msg->sender_idx].name); } } @@ -827,7 +829,7 @@ group_redo_last_applied (group); if (old_val < group->last_applied) { - gu_debug ("New COMMIT CUT %lld on %d after %lld from %d", + gu_debug ("New COMMIT CUT %lld on %ld after %lld from %d", (long long)group->last_applied, group->my_idx, (long long)gtid.seqno(), msg->sender_idx); return group->last_applied; @@ -1005,8 +1007,15 @@ return ret; } - if (gtid.uuid() == group->group_uuid && - gtid.seqno() > group->vote_result.seqno) + /* If either group-wide vote seqno or last applied are greater than the + request seqno, the vote has either happened already or there was no + need (i.e. all other members had a success). */ + gcs_seqno_t const min_seqno = + group->quorum.gcs_proto_ver >= 4 + ? std::max(group->last_applied, group->vote_result.seqno) + : group->vote_result.seqno; + + if (gtid.uuid() == group->group_uuid && gtid.seqno() > min_seqno) { const char* const data (gcs::core::CodeMsg::serial_size() < msg->size ? @@ -1019,7 +1028,8 @@ << gtid << ',' << gu::PrintBase<>(code) << ": " << (code ? (data ? data : "(null)") : "Success"); - gcs_node_set_vote (&sender, gtid.seqno(), code); + gcs_node_set_vote (&sender, gtid.seqno(), code, + group->quorum.gcs_proto_ver); if (group_recount_votes(*group)) { @@ -1161,15 +1171,16 @@ } } - if (j == group->num) { - gu_warn ("Could not find peer: %s", peer_id); + if (j == group->num && strlen(peer_id)) { + /* This can happen if the 'peer' is no longer in group. */ + gu_info ("Could not find peer: %s", peer_id); } if (code < 0) { - gu_warn ("%d.%d (%s): State transfer %s %d.%d (%s) failed: %d (%s)", + gu_warn ("%d.%d (%s): State transfer %s %d.%d (%s) failed: %s", sender_idx, sender->segment, sender->name, st_dir, peer_idx, peer ? peer->segment : -1, peer_name, - (int)code, strerror((int)-code)); + gcs_state_transfer_error_str((int)-code)); if (from_donor && peer_idx == group->my_idx && GCS_NODE_STATE_JOINER == group->nodes[peer_idx].status) { @@ -1211,8 +1222,14 @@ gu_warn("Rejecting JOIN message from %d.%d (%s): new State Transfer" " required.", sender_idx, sender->segment, sender->name); } - else { - // should we freak out and throw an error? + else if (GCS_NODE_STATE_SYNCED != sender->status && + GCS_NODE_STATE_JOINED != sender->status) { + /* According to comments in gcs_join(), sending of JOIN messages + * is always allowed when not in JOINER state. This may lead to + * duplicate joins of which some can be received in JOINED or + * SYNCED state. This is expected, so the warning is not printed if + * the state is JOINED or SYNCED, but we'll keep it for other + * states to catch possible errors in sender logic. */ gu_warn("Protocol violation. JOIN message sender %d.%d (%s) is not " "in state transfer (%s). Message ignored.", sender_idx, sender->segment, sender->name, @@ -1316,7 +1333,7 @@ /* Have not found suitable donor in the same segment. */ if (!hnss && donor >= 0) { if (joiner_idx == group->my_idx) { - gu_warn ("There are no nodes in the same segment that will ever " + gu_info ("There are no nodes in the same segment that will ever " "be able to become donors, yet there is a suitable donor " "outside. Will use that one."); } @@ -1658,7 +1675,6 @@ return donor_idx; } - /*! * Selects and returns the index of state transfer donor, if available. * Updates donor and joiner status if state transfer is possible @@ -1734,12 +1750,24 @@ assert(true == desync); } } - else { - gu_warn ("Member %d.%d (%s) requested state transfer from '%s', " - "but it is impossible to select State Transfer donor: %s", - joiner_idx, group->nodes[joiner_idx].segment, - group->nodes[joiner_idx].name, - required_donor ? donor_string : "*any*", strerror (-donor_idx)); + else if (-donor_idx == EAGAIN) { + /* In case of EAGAIN the failure of selecting the donor is + * transient, and donor selection may succeed when the request is + * retried by the Joiner. Therefore print info level message + * instead of warning. */ + gu_info("Member %d.%d (%s) requested state transfer from '%s', " + "but it is impossible to select State Transfer donor: %s", + joiner_idx, group->nodes[joiner_idx].segment, + group->nodes[joiner_idx].name, + required_donor ? donor_string : "*any*", + gcs_state_transfer_error_str(-donor_idx)); + } else { + gu_warn("Member %d.%d (%s) requested state transfer from '%s', " + "but it is impossible to select State Transfer donor: %s", + joiner_idx, group->nodes[joiner_idx].segment, + group->nodes[joiner_idx].name, + required_donor ? donor_string : "*any*", + gcs_state_transfer_error_str(-donor_idx)); } return donor_idx; @@ -1750,7 +1778,7 @@ gcs_group_ignore_action (gcs_group_t* group, struct gcs_act_rcvd* act) { gu_debug("Ignoring action: buf: %p, len: %zd, type: %d, sender: %d, " - "seqno: %lld", act->act.buf, act->act.buf_len, act->act.type, + "seqno: %" PRId64, act->act.buf, act->act.buf_len, act->act.type, act->sender_idx, act->id); if (act->act.type <= GCS_ACT_CCHANGE) { diff -Nru galera-4-26.4.18/gcs/src/gcs_node.cpp galera-4-26.4.20/gcs/src/gcs_node.cpp --- galera-4-26.4.18/gcs/src/gcs_node.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_node.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -7,6 +7,7 @@ #include "gcs_node.hpp" #include "gcs_state_msg.hpp" #include +#include #include // gu::PrintBase @@ -131,18 +132,25 @@ void gcs_node_set_vote (gcs_node_t* const node, gcs_seqno_t const seqno, - int64_t const vote) + int64_t const vote, + int const gcs_proto) { assert(0 == vote || seqno >= node->last_applied); assert(seqno > node->vote_seqno); - gcs_seqno_t const min_seqno(std::max(node->last_applied, node->vote_seqno)); + gcs_seqno_t const min_seqno = + gcs_proto >= 4 + ? node->vote_seqno + : std::max(node->last_applied, node->vote_seqno); if (gu_unlikely(seqno <= min_seqno)) { gu_warn ("Received bogus VOTE message: %lld.%0llx, from node %s, " "expected > %lld. Ignoring.", (long long)seqno, (long long)vote, node->id, (long long)min_seqno); + /* we should not be here: gcs_group_handle_vote_msg() should have + * taken care of it. */ + assert(0); } else { node->vote_seqno = seqno; @@ -185,10 +193,10 @@ else { // gap in sequence numbers, needs a snapshot, demote status if (node->status > GCS_NODE_STATE_PRIM) { - gu_info ("'%s' demoted %s->PRIMARY due to gap in history: " - "%lld - %lld", - node->name, gcs_node_state_to_str(node->status), - node_act_id, quorum->act_id); + gu_info("'%s' demoted %s->PRIMARY due to gap in history: " + "%" PRId64 " - %" PRId64, + node->name, gcs_node_state_to_str(node->status), + node_act_id, quorum->act_id); } node->status = GCS_NODE_STATE_PRIM; } diff -Nru galera-4-26.4.18/gcs/src/gcs_node.hpp galera-4-26.4.20/gcs/src/gcs_node.hpp --- galera-4-26.4.18/gcs/src/gcs_node.hpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_node.hpp 2024-07-30 05:28:41.000000000 +0000 @@ -115,9 +115,11 @@ { assert(seqno >= 0); if (gu_unlikely(seqno <= node->last_applied)) { - gu_warn ("Received bogus LAST message: %lld from node %s, " - "expected > %lld. Ignoring.", - (long long)seqno, node->id, (long long)node->last_applied); + if (node->count_last_applied) { + gu_warn("Received bogus LAST message: %lld from node %s, " + "expected > %lld. Ignoring.", + (long long)seqno, node->id, (long long)node->last_applied); + } } else { node->last_applied = seqno; @@ -125,7 +127,8 @@ } extern void -gcs_node_set_vote (gcs_node_t* node, gcs_seqno_t seqno, int64_t vote); +gcs_node_set_vote (gcs_node_t* node, gcs_seqno_t seqno, int64_t vote, + int gcs_ptoto); static inline gcs_seqno_t gcs_node_get_last_applied (gcs_node_t* node) diff -Nru galera-4-26.4.18/gcs/src/gcs_sm.cpp galera-4-26.4.20/gcs/src/gcs_sm.cpp --- galera-4-26.4.18/gcs/src/gcs_sm.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_sm.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -127,7 +127,7 @@ gu_mutex_unlock (&sm->lock); - if (ret) { gu_error ("Can't open send monitor: wrong state %d", ret); } + if (ret) { gu_error ("Can't open send monitor: wrong state %ld", ret); } return ret; } diff -Nru galera-4-26.4.18/gcs/src/gcs_state_msg.cpp galera-4-26.4.20/gcs/src/gcs_state_msg.cpp --- galera-4-26.4.18/gcs/src/gcs_state_msg.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/gcs_state_msg.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -568,11 +568,11 @@ state_report_uuids (buf, buf_len, states, states_num, GCS_NODE_STATE_NON_PRIM); #ifdef GCS_CORE_TESTING - gu_warn ("Quorum: No node with complete state:\n%s", buf); + gu_info ("Quorum: No node with complete state:\n%s", buf); #else /* Print buf into stderr in order to message truncation * of application logger. */ - gu_warn ("Quorum: No node with complete state:"); + gu_info ("Quorum: No node with complete state:"); fprintf(stderr, "%s\n", buf); #endif /* GCS_CORE_TESTING */ gu_free (buf); @@ -592,7 +592,8 @@ if (buf) { state_report_uuids (buf, buf_len, states, states_num, GCS_NODE_STATE_DONOR); - gu_fatal("Quorum impossible: conflicting group UUIDs:\n%s"); + gu_fatal("Quorum impossible: conflicting group UUIDs:\n%s", + buf); gu_free (buf); } else { diff -Nru galera-4-26.4.18/gcs/src/unit_tests/CMakeLists.txt galera-4-26.4.20/gcs/src/unit_tests/CMakeLists.txt --- galera-4-26.4.18/gcs/src/unit_tests/CMakeLists.txt 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/unit_tests/CMakeLists.txt 2024-07-30 05:28:41.000000000 +0000 @@ -35,6 +35,7 @@ ../gcs_params.cpp gcs_fc_test.cpp ../gcs_fc.cpp + ../gcs_error.cpp ) target_compile_definitions(gcs_tests diff -Nru galera-4-26.4.18/gcs/src/unit_tests/SConscript galera-4-26.4.20/gcs/src/unit_tests/SConscript --- galera-4-26.4.18/gcs/src/unit_tests/SConscript 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/unit_tests/SConscript 2024-07-30 05:28:41.000000000 +0000 @@ -56,6 +56,7 @@ ../gcs_params.cpp gcs_fc_test.cpp ../gcs_fc.cpp + ../gcs_error.cpp ''') diff -Nru galera-4-26.4.18/gcs/src/unit_tests/gcs_core_test.cpp galera-4-26.4.20/gcs/src/unit_tests/gcs_core_test.cpp --- galera-4-26.4.18/gcs/src/unit_tests/gcs_core_test.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/unit_tests/gcs_core_test.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -513,7 +513,7 @@ while (i--) { long frags = (act_size - 1)/FRAG_SIZE + 1; - gu_info ("Iteration %ld: act: %s, size: %zu, frags: %ld", + gu_info ("Iteration %ld: act: %p, size: %zu, frags: %ld", i, act, act_size, frags); ck_assert(!CORE_SEND_START (&act_s)); diff -Nru galera-4-26.4.18/gcs/src/unit_tests/gcs_test_utils.cpp galera-4-26.4.20/gcs/src/unit_tests/gcs_test_utils.cpp --- galera-4-26.4.18/gcs/src/unit_tests/gcs_test_utils.cpp 2024-03-22 11:54:21.000000000 +0000 +++ galera-4-26.4.20/gcs/src/unit_tests/gcs_test_utils.cpp 2024-07-30 05:28:41.000000000 +0000 @@ -53,7 +53,7 @@ node_name, inc_addr, gver, rver, aver)); if (err) { - gu_throw_error(-err) << "GcsGroup init failed"; + gu_throw_error(-err) << "GcsGroup init failed: " << -err; } initialized_ = true; @@ -452,7 +452,7 @@ int ret = gcs_group_handle_state_request(nodes[i]->group(), &req); if (ret < 0) { // don't fail here, we may want to test negatives - gu_error (ret < 0, "Handling state request to '%s' failed: %d (%s)", + gu_error ("Handling state request to '%s' failed: %d (%s)", donor_name, ret, strerror (-ret)); return ret; } diff -Nru galera-4-26.4.18/scripts/packages/codership-galera.spec galera-4-26.4.20/scripts/packages/codership-galera.spec --- galera-4-26.4.18/scripts/packages/codership-galera.spec 2024-03-22 11:54:24.000000000 +0000 +++ galera-4-26.4.20/scripts/packages/codership-galera.spec 2024-07-30 05:28:43.000000000 +0000 @@ -21,7 +21,7 @@ %define name galera-4 %define wsrep_api 26 -%{!?version: %define version 26.4.18} +%{!?version: %define version 26.4.20} %{!?release: %define release 1} %define copyright Copyright 2007-2020 Codership Oy. All rights reserved. Use is subject to license terms under GPLv2 license. %define libs %{_libdir}/%{name} diff -Nru galera-4-26.4.18/wsrep/src/wsrep_api.h galera-4-26.4.20/wsrep/src/wsrep_api.h --- galera-4-26.4.18/wsrep/src/wsrep_api.h 2024-03-22 11:54:22.000000000 +0000 +++ galera-4-26.4.20/wsrep/src/wsrep_api.h 2024-07-30 05:28:42.000000000 +0000 @@ -158,7 +158,7 @@ #endif /* __cplusplus */ /*! undefined seqno */ -#define WSREP_SEQNO_UNDEFINED (-1) +static wsrep_seqno_t const WSREP_SEQNO_UNDEFINED = -1; /*! wsrep provider status codes */ diff -Nru galera-4-26.4.18/wsrep/src/wsrep_loader.c galera-4-26.4.20/wsrep/src/wsrep_loader.c --- galera-4-26.4.18/wsrep/src/wsrep_loader.c 2024-03-22 11:54:22.000000000 +0000 +++ galera-4-26.4.20/wsrep/src/wsrep_loader.c 2024-07-30 05:28:42.000000000 +0000 @@ -165,7 +165,14 @@ return ret; } - if (!(dlh = dlopen(spec, RTLD_NOW | RTLD_LOCAL))) { + int open_flags = RTLD_NOW | RTLD_LOCAL; +#ifdef __SANITIZE_ADDRESS__ + /* Keep the shared object to allow ASAN resolve symbols and report + * memleaks. This also suppresses some false positives. */ + open_flags |= RTLD_NODELETE; +#endif /* __SANITIZE_ADDRESS__ */ + + if (!(dlh = dlopen(spec, open_flags))) { snprintf(msg, msg_len, "wsrep_load(): dlopen(): %s", dlerror()); logger (WSREP_LOG_ERROR, msg); ret = EINVAL; diff -Nru galera-4-26.4.18/wsrep/src/wsrep_node_isolation.h galera-4-26.4.20/wsrep/src/wsrep_node_isolation.h --- galera-4-26.4.18/wsrep/src/wsrep_node_isolation.h 1970-01-01 00:00:00.000000000 +0000 +++ galera-4-26.4.20/wsrep/src/wsrep_node_isolation.h 2024-07-30 05:28:42.000000000 +0000 @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2024 Codership Oy + * + * This file is part of wsrep-API. + * + * Wsrep-API is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-API is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-API. If not, see . + */ + +#ifndef WSREP_NODE_ISOLATION_H +#define WSREP_NODE_ISOLATION_H + +/** @file wsrep_node_isolation.h + * + * This file defines and interface to isolate the node from + * the rest of the cluster. The purpose of isolation is to shut + * down all communication with the rest of the cluster in case + * of node failure where the node cannot continue reliably anymore, + * e.g. in case of handling a signal which will terminate the process. + */ + +/** + * Mode of node isolation. + */ +enum wsrep_node_isolation_mode +{ + /** Node is not isolated. */ + WSREP_NODE_ISOLATION_NOT_ISOLATED, + /** Node is isolated from the rest of the cluster on network + * level. All ongoing network connections will be terminated and + * no new connections are accepted. */ + WSREP_NODE_ISOLATION_ISOLATED, + /** As WSREP_NODE_ISOLATION_ON, but also force the provider + * to deliver view with status WSREP_VIEW_DISCONNECTED. */ + WSREP_NODE_ISOLATION_FORCE_DISCONNECT, +}; + +enum wsrep_node_isolation_result +{ + /** Setting the isolation mode was successful. */ + WSREP_NODE_ISOLATION_SUCCESS, + /** Invalid isolation mode was passed. */ + WSREP_NODE_ISOLATION_INVALID_VALUE +}; + +/** Set mode isolation mode according to give wsrep_node_isolation_mode + * enum. + * + * The implementation must be async signal safe to allow calling + * it from program signal handler. + * + * @param mode Mode to set. + * @return wsrep_node_isolation_result enum. + */ +typedef enum wsrep_node_isolation_result (*wsrep_node_isolation_mode_set_fn_v1)( + enum wsrep_node_isolation_mode mode); + +#define WSREP_NODE_ISOLATION_MODE_SET_V1 "wsrep_node_isolation_mode_set_v1" + +#endif /* WSREP_NODE_ISOLATION_H */