/*
 * Copyright (c) 2020, 2026, Oracle and/or its affiliates.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2.0,
 * as published by the Free Software Foundation.
 *
 * This program is designed to work with certain software (including
 * but not limited to OpenSSL) that is licensed under separate terms,
 * as designated in a particular file or component or in included license
 * documentation.  The authors of MySQL hereby grant you an additional
 * permission to link the program and your derivative works with the
 * separately licensed software that they have either included with
 * the program or referenced in the documentation.
 *
 * This program is distributed in the hope that it will be useful,  but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
 * the GNU General Public License, version 2.0, for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "modules/util/dump/dumper.h"

#include <algorithm>
#include <chrono>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <list>
#include <set>
#include <shared_mutex>
#include <type_traits>
#include <utility>

#include <mysqld_error.h>

#include <rapidjson/document.h>
#include <rapidjson/pointer.h>
#include <rapidjson/prettywriter.h>
#include <rapidjson/stringbuffer.h>

#include "mysqlshdk/include/scripting/shexcept.h"
#include "mysqlshdk/include/shellcore/console.h"
#include "mysqlshdk/include/shellcore/shell_init.h"
#include "mysqlshdk/include/shellcore/shell_options.h"
#include "mysqlshdk/libs/db/utils/utils.h"
#include "mysqlshdk/libs/mysql/binlog_utils.h"
#include "mysqlshdk/libs/mysql/gtid_utils.h"
#include "mysqlshdk/libs/mysql/replication.h"
#include "mysqlshdk/libs/textui/textui.h"
#include "mysqlshdk/libs/utils/debug.h"
#include "mysqlshdk/libs/utils/thread_pool.h"

#include "mysqlshdk/libs/oci/oci_bucket_config.h"
#include "mysqlshdk/libs/oci/oci_par.h"
#include "mysqlshdk/libs/storage/utils.h"
#include "mysqlshdk/libs/utils/fault_injection.h"
#include "mysqlshdk/libs/utils/option_tracker.h"
#include "mysqlshdk/libs/utils/profiling.h"
#include "mysqlshdk/libs/utils/rate_limit.h"
#include "mysqlshdk/libs/utils/strformat.h"
#include "mysqlshdk/libs/utils/utils_account.h"
#include "mysqlshdk/libs/utils/utils_file.h"
#include "mysqlshdk/libs/utils/utils_general.h"
#include "mysqlshdk/libs/utils/utils_json.h"
#include "mysqlshdk/libs/utils/utils_lexing.h"
#include "mysqlshdk/libs/utils/utils_mysql_parsing.h"
#include "mysqlshdk/libs/utils/utils_path.h"
#include "mysqlshdk/libs/utils/utils_sqlstring.h"
#include "mysqlshdk/libs/utils/utils_string.h"

#include "modules/mod_utils.h"
#include "modules/util/common/data_masking.h"
#include "modules/util/common/dump/constants.h"
#include "modules/util/common/dump/dump_version.h"
#include "modules/util/common/dump/utils.h"
#include "modules/util/common/utils.h"
#include "modules/util/dump/compatibility_issue.h"
#include "modules/util/dump/compatibility_option.h"
#include "modules/util/dump/console_with_progress.h"
#include "modules/util/dump/decimal.h"
#include "modules/util/dump/dialect_dump_writer.h"
#include "modules/util/dump/dump_errors.h"
#include "modules/util/dump/indexes.h"
#include "modules/util/dump/schema_dumper.h"
#include "modules/util/dump/text_dump_writer.h"
#include "modules/util/upgrade_check.h"
#include "modules/util/upgrade_checker/common.h"
#include "modules/util/upgrade_checker/upgrade_check_config.h"
#include "modules/util/upgrade_checker/upgrade_check_options.h"

namespace mysqlsh {
namespace dump {

using mysqlshdk::storage::Mode;
using mysqlshdk::storage::backend::Memory_file;
using mysqlshdk::utils::k_shell_version;
using mysqlshdk::utils::Version;
using Row = std::vector<std::string>;

namespace {

static constexpr const int k_mysql_server_net_write_timeout = 30 * 60;
static constexpr const int k_mysql_server_wait_timeout = 365 * 24 * 60 * 60;

// 255 characters total:
// - 225 - base name
// -   5 - base name ordinal number
// -   2 - '@@'
// -   5 - chunk ordinal number
// -  10 - extension
// -   8 - '.dumping' extension
static constexpr std::size_t k_max_basename_length = 225;

FI_DEFINE(dumper, [](const mysqlshdk::utils::FI::Args &args) {
  const auto op = args.get_string("op");

  if ("WORKER_SLEEP_AT_START" == op) {
    shcore::sleep_ms(args.get_int("sleep"));
  }
});

class Scoped_session final {
 public:
  Scoped_session() = delete;

  explicit Scoped_session(
      shcore::Synchronized_queue<std::shared_ptr<mysqlshdk::db::ISession>>
          &pool)
      : m_pool(pool), m_session(pool.pop()) {}

  Scoped_session(const Scoped_session &) = delete;
  Scoped_session(Scoped_session &&) = delete;

  Scoped_session &operator=(const Scoped_session &) = delete;
  Scoped_session &operator=(Scoped_session &&) = delete;

  ~Scoped_session() {
    if (m_session) {
      m_pool.push(std::move(m_session));
    }
  }

  const std::shared_ptr<mysqlshdk::db::ISession> &get() const noexcept {
    return m_session;
  }

 private:
  shcore::Synchronized_queue<std::shared_ptr<mysqlshdk::db::ISession>> &m_pool;
  std::shared_ptr<mysqlshdk::db::ISession> m_session;
};

std::string quote_value(const std::string &value, mysqlshdk::db::Type type) {
  if (is_string_type(type)) {
    return shcore::quote_sql_string(value);
  } else if (mysqlshdk::db::Type::Decimal == type) {
    return "'" + value + "'";
  } else {
    return value;
  }
}

auto refs(const std::string &s) {
  return rapidjson::StringRef(s.c_str(), s.length());
}

std::string to_string(rapidjson::Document *doc) {
  rapidjson::StringBuffer buffer;
  rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
  doc->Accept(writer);
  return std::string{buffer.GetString(), buffer.GetSize()};
}

void write_json(std::unique_ptr<mysqlshdk::storage::IFile> file,
                rapidjson::Document *doc) {
  const auto json = to_string(doc);
  file->open(Mode::WRITE);
  file->write(json.c_str(), json.length());
  file->close();
}

std::string quote(const std::string &schema, const std::string &table) {
  return shcore::quote_identifier(schema) + "." +
         shcore::quote_identifier(table);
}

Row fetch_row(const mysqlshdk::db::IRow *row) {
  Row result;

  if (row) {
    for (uint32_t i = 0, size = row->num_fields(); i < size; ++i) {
      assert(!row->is_null(i));
      result.emplace_back(row->get_as_string(i));
    }
  }

  return result;
}

int64_t to_int64_t(const std::string &s) { return std::stoll(s); }

uint64_t to_uint64_t(const std::string &s) { return std::stoull(s); }

using mysqlshdk::mysql::Gtid;
using mysqlshdk::mysql::Gtid_range;
using mysqlshdk::mysql::Gtid_set;

bool check_if_transactions_are_ddl_safe(
    const mysqlshdk::mysql::IInstance &instance,
    const common::Binlog::File &from, const common::Binlog::File &to,
    const Gtid_set &gtid_set = {}) {
  std::vector<Gtid_range> gtid_ranges;
  uint64_t count = 0;

  gtid_set.enumerate_ranges([&gtid_ranges, &count](const Gtid_range &range) {
    gtid_ranges.emplace_back(range);
    count += range.count();
  });

  const auto console = current_console();
  console->print_note("Checking" + (count ? " " + std::to_string(count) : "") +
                      " recent transactions for schema changes, use the "
                      "'skipConsistencyChecks' option to skip this check.");

  std::vector<std::string> binlogs;

  if (from.name == to.name) {
    binlogs.emplace_back(from.name);
  } else {
    auto all_binlogs = list_binlogs(instance);
    const auto end =
        std::find(all_binlogs.rbegin(), all_binlogs.rend(), to.name);
    const auto begin = std::find(end, all_binlogs.rend(), from.name);
    binlogs.insert(
        binlogs.end(),
        std::make_move_iterator(begin == all_binlogs.rend()
                                    ? all_binlogs.begin()
                                    : begin.base() - 1),
        std::make_move_iterator(end == all_binlogs.rend() ? all_binlogs.begin()
                                                          : end.base()));
  }

  const auto include_gtid = [&gtid_ranges](const Gtid &gtid) {
    if (gtid_ranges.empty()) {
      return true;
    }

    auto gtid_range = Gtid_range::from_gtid(gtid);
    if (!gtid_range) return false;
    assert(gtid_range.begin == gtid_range.end);

    for (const auto &range : gtid_ranges) {
      if (range.uuid_tag == gtid_range.uuid_tag &&
          range.begin <= gtid_range.begin && gtid_range.begin <= range.end) {
        return true;
      }
    }

    return false;
  };

  const auto is_ddl_safe = [](const std::string &transaction) {
    std::istringstream stream(transaction);
    bool is_safe = true;

    mysqlshdk::utils::iterate_sql_stream(
        &stream, transaction.length(),
        [&is_safe](std::string_view sql, std::string_view, size_t, size_t) {
          mysqlshdk::utils::SQL_iterator it(sql);
          const auto token = it.next_token();

          if (shcore::str_caseeq(token, "ALTER", "CREATE", "DROP", "GRANT",
                                 "REVOKE") ||
              (shcore::str_caseeq(token, "RENAME", "TRUNCATE") &&
               shcore::str_caseeq(it.next_token(), "TABLE"))) {
            is_safe = false;
            return false;
          }

          return true;
        },
        [](std::string_view err) {
          throw std::runtime_error(
              "Failed to check if transaction is DDL safe: " +
              std::string{err});
        });

    return is_safe;
  };

  bool is_safe = true;

  for (const auto &binlog : binlogs) {
    std::optional<uint64_t> start_position;

    if (binlog == from.name) {
      start_position = from.position;
    }

    const auto last_file = binlog == to.name;

    list_binlog_events(
        instance, binlog,
        [&include_gtid, &is_ddl_safe, &is_safe, last_file, end = to.position](
            const Gtid &gtid, const mysqlshdk::mysql::Binlog_event &event) {
          if (last_file && event.pos >= end) {
            return false;
          }

          if (event.info.empty() ||
              !shcore::str_caseeq(event.event_type, "Query")) {
            return true;
          }

          if (include_gtid(gtid) && !is_ddl_safe(event.info)) {
            is_safe = false;
            return false;
          }

          return true;
        },
        start_position);

    if (last_file || !is_safe) {
      break;
    }
  }

  if (!is_safe) {
    console->print_warning(
        "DDL changes detected during DDL dump without a lock.");
  }

  return is_safe;
}

void append_capability_metadata(
    const std::unordered_set<Capability> &capabilities,
    shcore::JSON_dumper *json) {
  json->append("capabilities");
  json->start_array();

  for (const auto &c : capabilities) {
    json->start_object();

    json->append("id", capability::id(c));
    json->append("description", capability::description(c));
    json->append("versionRequired", capability::version_required(c).get_base());

    json->end_object();
  }

  json->end_array();
}

inline bool is_encoding_needed(Dump_writer::Encoding_type encoding) {
  return Dump_writer::Encoding_type::NONE != encoding;
}

void ensure_prefix_par(std::string *s) {
  if (mysqlshdk::oci::PAR_type::GENERAL == mysqlshdk::oci::parse_par(*s)) {
    *s += '/';
  }
}

}  // namespace

class Dumper::Memory_dumper final {
 public:
  Memory_dumper() = delete;

  explicit Memory_dumper(Schema_dumper *dumper)
      : m_dumper(dumper), m_file("/dev/null") {}

  Memory_dumper(const Memory_dumper &) = delete;
  Memory_dumper(Memory_dumper &&) = default;

  Memory_dumper &operator=(const Memory_dumper &) = delete;
  Memory_dumper &operator=(Memory_dumper &&) = default;

  ~Memory_dumper() = default;

  const std::vector<Compatibility_issue> &dump(
      const std::function<void(Memory_dumper *)> &func) {
    m_issues.clear();

    m_file.open(Mode::WRITE);
    func(this);
    m_file.close();

    return issues();
  }

  const std::vector<Compatibility_issue> &issues() const { return m_issues; }

  const std::string &content() const { return m_file.content(); }

  template <typename... Args>
  void dump(std::vector<Compatibility_issue> (Schema_dumper::*func)(
                Schema_dumper::IFile *, const std::remove_cvref_t<Args> &...),
            Args &&...args) {
    auto issues = (m_dumper->*func)(&m_file, std::forward<Args>(args)...);

    m_issues.insert(m_issues.end(), std::make_move_iterator(issues.begin()),
                    std::make_move_iterator(issues.end()));
  }

  template <typename... Args>
  void dump(void (Schema_dumper::*func)(Schema_dumper::IFile *,
                                        const std::remove_cvref_t<Args> &...),
            Args &&...args) {
    (m_dumper->*func)(&m_file, std::forward<Args>(args)...);
  }

 private:
  Schema_dumper *m_dumper;
  Memory_file m_file;
  std::vector<Compatibility_issue> m_issues;
};
class Dumper::Dump_writer_controller {
 public:
  using Create_file = std::function<std::unique_ptr<mysqlshdk::storage::IFile>(
      const std::string &)>;

  Dump_writer_controller() = delete;

  Dump_writer_controller(const Dump_writer_controller &) = delete;
  Dump_writer_controller(Dump_writer_controller &&) = default;

  Dump_writer_controller &operator=(const Dump_writer_controller &) = delete;
  Dump_writer_controller &operator=(Dump_writer_controller &&) = default;

  virtual ~Dump_writer_controller() = default;

  inline const Dump_write_result &total_stats() const {
    return m_total_written;
  }

  inline const Dump_write_result &progress_stats() const {
    return m_written_per_update;
  }

  inline void reset_progress() { m_written_per_update.reset(); }

  inline uint64_t longest_row() const { return m_longest_row; }

  inline const std::string &output_filename() const {
    return m_output_filename;
  }

  virtual void prepare_for_writing() {
    assert(m_output);

    m_writer->set_output_file(m_output);

    if (m_create_index) {
      m_writer->set_index_file(m_create_index(m_output_filename + ".idx"));
    }

    if (!m_output->is_open()) {
      m_output->open(Mode::WRITE);
    }

    m_writer->open();
  }

  virtual Dump_write_result start_writing(
      const std::vector<mysqlshdk::db::Column> &metadata,
      const std::vector<Dump_writer::Encoding_type> &pre_encoded_columns) {
    m_writer->initialize_writing(metadata, pre_encoded_columns);
    return update_stats(m_writer->write_preamble());
  }

  virtual Dump_write_result write_row(const mysqlshdk::db::IRow *row) {
    assert(m_output);
    return update_stats(m_writer->write_row(row));
  }

  virtual Dump_write_result finish_writing() {
    assert(m_output);

    auto result = m_writer->write_postamble();
    m_writer->close();

    if (m_close_output && m_output->is_open()) {
      m_output->close();

      // if file is compressed, the final file size may differ from the bytes
      // written so far, as i.e. bytes were not flushed until the whole block
      // was ready
      result.write_bytes(m_output->file_size() -
                         m_total_written.bytes_written() -
                         result.bytes_written());
    }

    return update_stats(result);
  }

  virtual void update_uncompressed_file_size(
      std::unordered_map<std::string, uint64_t> *stats) const {
    (*stats)[output_filename()] += m_total_written.data_bytes();
  }

 protected:
  explicit Dump_writer_controller(std::unique_ptr<Dump_writer> writer)
      : m_writer(std::move(writer)) {}

  void set_output(mysqlshdk::storage::IFile *output) { m_output = output; }

  void set_output_filename(const std::string &name) {
    m_output_filename = name;
  }

  void write_index(Create_file create_index) {
    m_create_index = std::move(create_index);
  }

  void close_output() { m_close_output = true; }

  Dump_write_result update_stats(Dump_write_result result) {
    m_total_written += result;
    m_written_per_update += result;

    if (result.rows_written() && result.data_bytes() > m_longest_row) {
      m_longest_row = result.data_bytes();
    }

    return result;
  }

 private:
  mysqlshdk::storage::IFile *m_output = nullptr;
  std::string m_output_filename;
  std::unique_ptr<Dump_writer> m_writer;
  Create_file m_create_index;
  bool m_close_output = false;

  Dump_write_result m_total_written;
  Dump_write_result m_written_per_update;
  uint64_t m_longest_row = 0;
};

class Dumper::Single_file_writer_controller : public Dump_writer_controller {
 public:
  Single_file_writer_controller() = delete;

  Single_file_writer_controller(std::unique_ptr<Dump_writer> writer,
                                mysqlshdk::storage::IFile *output)
      : Dump_writer_controller(std::move(writer)) {
    set_output(output);
    set_output_filename(output->filename());
  }

  Single_file_writer_controller(const Single_file_writer_controller &) = delete;
  Single_file_writer_controller(Single_file_writer_controller &&) = default;

  Single_file_writer_controller &operator=(
      const Single_file_writer_controller &) = delete;
  Single_file_writer_controller &operator=(Single_file_writer_controller &&) =
      default;

  ~Single_file_writer_controller() = default;
};

class Dumper::Default_writer_controller : public Dump_writer_controller {
 public:
  Default_writer_controller() = delete;

  Default_writer_controller(std::unique_ptr<Dump_writer> writer,
                            Create_file create_file, Create_file create_index,
                            const std::string &filename, bool add_suffix)
      : Dump_writer_controller(std::move(writer)),
        m_create_file(std::move(create_file)),
        m_add_suffix(add_suffix) {
    set_output_filename(filename);
    write_index(std::move(create_index));
    close_output();
  }

  Default_writer_controller(const Default_writer_controller &) = delete;
  Default_writer_controller(Default_writer_controller &&) = default;

  Default_writer_controller &operator=(const Default_writer_controller &) =
      delete;
  Default_writer_controller &operator=(Default_writer_controller &&) = default;

  ~Default_writer_controller() = default;

  void prepare_for_writing() override {
    auto filename = output_filename();

    if (m_add_suffix) {
      filename += k_dump_in_progress_ext;
    }

    m_output = m_create_file(filename);
    set_output(m_output.get());

    Dump_writer_controller::prepare_for_writing();
  }

  Dump_write_result finish_writing() override {
    auto result = Dump_writer_controller::finish_writing();

    if (m_add_suffix) {
      m_output->rename(output_filename());
    }

    m_output.reset();

    return result;
  }

 private:
  static constexpr std::string_view k_dump_in_progress_ext = ".dumping";

  Create_file m_create_file;
  bool m_add_suffix;
  std::unique_ptr<mysqlshdk::storage::IFile> m_output;
};

class Dumper::Multi_file_writer_controller : public Dump_writer_controller {
 public:
  using Create_controller =
      std::function<std::unique_ptr<Dump_writer_controller>(
          const std::string &)>;

  Multi_file_writer_controller() = delete;

  Multi_file_writer_controller(Create_controller create_controller,
                               const std::string &basename,
                               const std::string &extension,
                               uint64_t bytes_per_file)
      : Dump_writer_controller(std::unique_ptr<Dump_writer>{}),
        m_create_controller(std::move(create_controller)),
        m_extension(extension),
        m_bytes_per_file(bytes_per_file) {
    set_output_filename(basename);
  }

  Multi_file_writer_controller(const Multi_file_writer_controller &) = delete;
  Multi_file_writer_controller(Multi_file_writer_controller &&) = default;

  Multi_file_writer_controller &operator=(
      const Multi_file_writer_controller &) = delete;
  Multi_file_writer_controller &operator=(Multi_file_writer_controller &&) =
      default;

  ~Multi_file_writer_controller() = default;

  void prepare_for_writing() override { create_controller(false); }

  Dump_write_result start_writing(
      const std::vector<mysqlshdk::db::Column> &metadata,
      const std::vector<Dump_writer::Encoding_type> &pre_encoded_columns)
      override {
    m_metadata = metadata;
    m_pre_encoded_columns = pre_encoded_columns;

    return start_writing();
  }

  Dump_write_result write_row(const mysqlshdk::db::IRow *row) override {
    Dump_write_result result;

    if (!m_controller) {
      result += initialize_controller(false);
    }

    result += update_stats(m_controller->write_row(row));

    if (m_controller->total_stats().data_bytes() >= m_bytes_per_file) {
      result += finalize_controller();
    }

    return result;
  }

  Dump_write_result finish_writing() override {
    Dump_write_result result;

    if (m_controller) {
      result += finalize_controller();
    }

    // create an empty final chunk
    result += initialize_controller(true);
    result += finalize_controller();

    return result;
  }

  void update_uncompressed_file_size(
      std::unordered_map<std::string, uint64_t> *stats) const override {
    for (const auto &file : m_file_stats) {
      (*stats)[file.first] += file.second.data_bytes();
    }
  }

 private:
  void create_controller(bool last_chunk) {
    m_controller = m_create_controller(common::get_table_data_filename(
        output_filename(), m_extension, m_index++, last_chunk));
    m_controller->prepare_for_writing();
  }

  Dump_write_result start_writing() {
    return update_stats(
        m_controller->start_writing(m_metadata, m_pre_encoded_columns));
  }

  Dump_write_result initialize_controller(bool last_chunk) {
    create_controller(last_chunk);
    return start_writing();
  }

  Dump_write_result finalize_controller() {
    auto result = update_stats(m_controller->finish_writing());
    m_file_stats.emplace(m_controller->output_filename(),
                         m_controller->total_stats());
    m_controller.reset();
    return result;
  }

  Create_controller m_create_controller;
  std::string m_extension;
  uint64_t m_bytes_per_file;
  std::size_t m_index = 0;
  std::unique_ptr<Dump_writer_controller> m_controller;
  std::vector<mysqlshdk::db::Column> m_metadata;
  std::vector<Dump_writer::Encoding_type> m_pre_encoded_columns;
  std::unordered_map<std::string, Dump_write_result> m_file_stats;
};

void Dumper::Output_config::init(
    const import_table::Dialect &d, mysqlshdk::storage::Compression c,
    const mysqlshdk::storage::Compression_options &co) {
  dialect = d;
  compression = c;
  compression_options = co;

  if (import_table::Dialect::default_() == d) {
    writer_creator = []() { return std::make_unique<Default_dump_writer>(); };
    data_file_extension = "tsv";
  } else if (import_table::Dialect::json() == d) {
    writer_creator = []() { return std::make_unique<Json_dump_writer>(); };
    data_file_extension = "json";
  } else if (import_table::Dialect::csv() == d) {
    writer_creator = []() { return std::make_unique<Csv_dump_writer>(); };
    data_file_extension = "csv";
  } else if (import_table::Dialect::tsv() == d) {
    writer_creator = []() { return std::make_unique<Tsv_dump_writer>(); };
    data_file_extension = "tsv";
  } else if (import_table::Dialect::csv_unix() == d) {
    writer_creator = []() { return std::make_unique<Csv_unix_dump_writer>(); };
    data_file_extension = "csv";
  } else if (import_table::Dialect::csv_rfc_unix() == d) {
    writer_creator = []() {
      return std::make_unique<Csv_rfc_unix_dump_writer>();
    };
    data_file_extension = "csv";
  } else {
    writer_creator = [this]() {
      return std::make_unique<Text_dump_writer>(dialect);
    };
    data_file_extension = "txt";
  }

  data_file_extension += mysqlshdk::storage::get_extension(c);
}

void Dumper::Output_config::create_directory() {
  if (!dir->exists()) {
    dir->create();
  }
}

void Dumper::Output_config::fini() { dir.reset(); }

class Dumper::Table_worker final {
 public:
  enum class Exception_strategy { ABORT, CONTINUE };

  Table_worker() = delete;

  Table_worker(std::size_t id, Dumper *dumper, Exception_strategy strategy)
      : m_id(id),
        m_log_id(shcore::str_format("[Worker%03zu]: ", m_id)),
        m_dumper(dumper),
        m_strategy(strategy) {}

  Table_worker(const Table_worker &) = delete;
  Table_worker(Table_worker &&) = default;

  Table_worker &operator=(const Table_worker &) = delete;
  Table_worker &operator=(Table_worker &&) = delete;

  ~Table_worker() = default;

  void run() {
    std::string context;

    try {
      FI_TRIGGER_TRAP(dumper, mysqlshdk::utils::FI::Trigger_options(
                                  {{"op", "WORKER_SLEEP_AT_START"},
                                   {"id", std::to_string(m_id)}}));

      mysqlsh::Mysql_thread mysql_thread;
      while (true) {
        auto work = m_dumper->m_worker_tasks.pop();

        if (m_dumper->m_worker_interrupt.test()) {
          return;
        }

        if (!work.task) {
          break;
        }

        context = std::move(work.info);

        m_session = m_dumper->session_pool().pop();
        shcore::on_leave_scope session_releaser(
            [this]() { release_session(); });

        if (m_dumper->m_worker_interrupt.test()) {
          return;
        }

        work.task(this);

        if (m_dumper->m_worker_interrupt.test()) {
          return;
        }
      }
    } catch (const mysqlshdk::db::Error &e) {
      handle_exception(context, e.format().c_str());
    } catch (const shcore::Error &e) {
      // these are global errors and should not include a context
      if (SHERR_DUMP_CONSISTENCY_CHECK_FAILED == e.code() ||
          SHERR_DUMP_INVALID_GRANT_STATEMENT == e.code() ||
          SHERR_DUMP_COMPATIBILITY_ISSUES_FOUND == e.code()) {
        context.clear();
      }

      handle_exception(context, e.what());
    } catch (const std::exception &e) {
      handle_exception(context, e.what());
    } catch (...) {
      handle_exception(context, "Unknown exception");
    }
  }

  inline void write_schema_metadata(const Schema_info &schema) const {
    log_info("%sWriting metadata for schema %s", m_log_id.c_str(),
             schema.quoted_name.c_str());
    write_schema_metadata_impl(schema);
  }

  inline void write_table_metadata(const Table_task &table) const {
    log_info("%sWriting metadata for table %s", m_log_id.c_str(),
             table.quoted_name.c_str());
    write_table_metadata_impl(table);
  }

  inline void dump_schema_ddl(const Schema_info &schema) const {
    log_info("%sWriting DDL for schema %s", m_log_id.c_str(),
             schema.quoted_name.c_str());
    dump_schema_ddl_impl(schema);
  }

  inline void dump_table_ddl(const Schema_info &schema,
                             const Table_info &table) const {
    log_info("%sWriting DDL for table %s", m_log_id.c_str(),
             table.quoted_name.c_str());
    dump_table_ddl_impl(schema, table);
  }

  inline void dump_view_ddl(const Schema_info &schema,
                            const View_info &view) const {
    log_info("%sWriting DDL for view %s", m_log_id.c_str(),
             view.quoted_name.c_str());
    dump_view_ddl_impl(schema, view);
  }

  inline void create_table_data_tasks(const Table_task &table) const {
    log_debug("%sCreating data tasks for table %s", m_log_id.c_str(),
              table.quoted_name.c_str());

    ++m_dumper->m_num_threads_chunking;
    create_table_data_tasks_impl(table);
    --m_dumper->m_num_threads_chunking;
  }

  inline void dump_table_data(const Table_data_task &table) {
    log_debug(
        "%sDumping %s (%s) using boundary: %s, filter: %s", m_log_id.c_str(),
        table.task_name.c_str(), table.id.c_str(),
        table.boundary.length() ? table.boundary.c_str() : "NONE",
        table.extra_filter.length() ? table.extra_filter.c_str() : "NONE");

    ++m_dumper->m_num_threads_dumping;
    dump_table_data_impl(table);
    --m_dumper->m_num_threads_dumping;
  }

  inline void checksum_table_data(const Checksum_task &task) const {
    log_debug("%sComputing checksum of %s (%s)", m_log_id.c_str(),
              task.name.c_str(), task.id.c_str());

    ++m_dumper->m_num_threads_checksumming;
    checksum_table_data_impl(task);
    --m_dumper->m_num_threads_checksumming;
  }

 private:
  inline const std::shared_ptr<mysqlshdk::db::ISession> &session()
      const noexcept {
    assert(m_session);
    return m_session;
  }

  void release_session() {
    if (m_session) {
      m_dumper->session_pool().push(std::move(m_session));
    }
  }

  inline std::shared_ptr<mysqlshdk::db::IResult> query(
      const std::string &sql) const {
    return Dumper::query(session(), sql);
  }

  Dump_writer::Encoding_type encode_column(
      const mysqlsh::dump::Instance_cache::Table *table,
      const mysqlsh::dump::Instance_cache::Column *column,
      std::string *query) const {
    const auto encoding = m_dumper->encoding_type(table, column);
    const auto encoding_needed = is_encoding_needed(encoding);

    if (encoding_needed) {
      std::string_view function;

      switch (encoding) {
        case Dump_writer::Encoding_type::BASE64:
          function = "TO_BASE64";
          break;

        case Dump_writer::Encoding_type::HEX:
          function = "HEX";
          break;

        case Dump_writer::Encoding_type::VECTOR:
          function = "VECTOR_TO_STRING";
          break;

        case Dump_writer::Encoding_type::NONE:
          assert(false);
          break;
      }

      *query += function;
      *query += '(';
    }

    *query += column->quoted_name;

    if (encoding_needed) {
      *query += ')';
    }

    return encoding;
  }

  std::string prepare_query(
      const Table_data_task &table,
      std::vector<Dump_writer::Encoding_type> *out_pre_encoded_columns) const {
    out_pre_encoded_columns->reserve(table.info->columns.size());

    std::string query = "SELECT " + m_dumper->optimizer_hints(table.info);

    for (const auto &column : table.info->columns) {
      out_pre_encoded_columns->emplace_back(
          encode_column(table.info, column, &query));
      query += ',';
    }

    // remove last comma
    query.pop_back();

    query += " FROM ";
    query += table.quoted_name;

    if (!table.partitions.empty()) {
      query += " PARTITION (";
      query += shcore::str_join(
          table.partitions.begin(), table.partitions.end(), ",",
          [](const auto &p) { return p.info->quoted_name; });
      query += ')';
    }

    query += where(table);

    if (table.index.info) {
      query += " ORDER BY ";
      query += table.index.info->columns_sql();
    }

    query += ' ';
    query += m_dumper->get_query_comment(table, "dumping");

    return query;
  }

  void dump_table_data_impl(const Table_data_task &table) {
    mysqlshdk::utils::Duration duration;
    duration.start();

    std::vector<Dump_writer::Encoding_type> pre_encoded_columns;
    const auto full_query = prepare_query(table, &pre_encoded_columns);
    const auto controller = table.controller.get();

    try {
      controller->prepare_for_writing();

      if (Dry_run::DISABLED == m_dumper->m_options.dry_run_mode()) {
        mysqlshdk::utils::Rate_limit rate_limit{m_dumper->m_options.max_rate()};
        const auto result = query(full_query);

        controller->start_writing(result->get_metadata(), pre_encoded_columns);

        while (const auto row = result->fetch_one()) {
          if (m_dumper->m_worker_interrupt.test()) {
            return;
          }

          controller->write_row(row);

          constexpr uint64_t update_every = 2000;
          if (update_every == controller->progress_stats().rows_written()) {
            m_dumper->update_progress(controller->progress_stats());

            // we don't know how much data was read from the server, number of
            // bytes written to the dump file is a good approximation
            if (rate_limit.enabled()) {
              rate_limit.throttle(controller->progress_stats().data_bytes());
            }

            controller->reset_progress();
          }
        }
      }
    } catch (const mysqlshdk::db::Error &e) {
      log_error("%sFailed to dump %s (%s) using query: %s, error: %s",
                m_log_id.c_str(), table.task_name.c_str(), table.id.c_str(),
                full_query.c_str(), e.format().c_str());
      throw;
    }

    release_session();

    controller->finish_writing();

    duration.finish();

    log_info("%sDump of %s (%s) into '%s' took %f seconds, written %" PRIu64
             " rows (%" PRIu64 " bytes), longest row has %" PRIu64 " bytes",
             m_log_id.c_str(), table.task_name.c_str(), table.id.c_str(),
             controller->output_filename().c_str(), duration.seconds_elapsed(),
             controller->total_stats().rows_written(),
             controller->total_stats().data_bytes(), controller->longest_row());

    m_dumper->update_progress(controller->progress_stats());
    m_dumper->finish_writing(table.schema, table.name, controller);
    m_dumper->data_task_finished();
  }

  Table_data_task create_table_data_task(const Table_task &table,
                                         const std::string &filename,
                                         int64_t chunk = -1) const {
    Table_data_task data_task;

    data_task.task_name = table.task_name;
    data_task.name = table.name;
    data_task.quoted_name = table.quoted_name;
    data_task.schema = table.schema;
    data_task.info = table.info;
    data_task.index = table.index;
    data_task.output_config = table.output_config;
    data_task.partitions = table.partitions;
    data_task.extra_filter = table.extra_filter;
    data_task.chunk = chunk;

    if (!filename.empty()) {
      data_task.controller =
          m_dumper->table_dump_controller(table.output_config, filename);
    }

    return data_task;
  }

  void create_and_push_whole_table_data_task(const Table_task &table) const {
    Table_data_task data_task = create_table_data_task(
        table, common::get_table_data_filename(
                   table.basename, table.output_config->data_file_extension));

    data_task.id = "whole table";

    m_dumper->push_table_data_task(std::move(data_task));
  }

  void create_and_push_whole_table_data_in_chunks_task(
      const Table_task &table) const {
    Table_data_task data_task = create_table_data_task(table, {});

    data_task.id = "whole table split in chunks";
    data_task.controller = m_dumper->table_dump_multi_file_controller(
        table.output_config, table.basename);

    m_dumper->push_table_data_task(std::move(data_task));
  }

  void create_and_push_table_data_chunk_task(const Table_task &table,
                                             const std::string &boundary,
                                             const std::string &id,
                                             std::size_t idx,
                                             bool last_chunk) const {
    Table_data_task data_task = create_table_data_task(
        table,
        common::get_table_data_filename(
            table.basename, table.output_config->data_file_extension, idx,
            last_chunk),
        idx);

    data_task.id = "chunk " + id;

    if (!boundary.empty()) {
      data_task.boundary = "(" + boundary + ")";
    }

    if (0 == idx && table.index.info && !table.index.is_pke) {
      for (const auto &c : table.index.info->columns()) {
        if (c->nullable) {
          if (!data_task.boundary.empty()) {
            data_task.boundary += "OR";
          }

          data_task.boundary += "(" + c->quoted_name + " IS NULL)";
        }
      }

      data_task.boundary = "(" + data_task.boundary + ")";
    }

    m_dumper->push_table_data_task(std::move(data_task));
  }

  void checksum_table_data_impl(const Checksum_task &task) const {
    log_debug("%sComputing checksum of %s (%s)", m_log_id.c_str(),
              task.name.c_str(), task.id.c_str());

    m_dumper->checksum_task_started();

    if (Dry_run::DISABLED == m_dumper->m_options.dry_run_mode()) {
      task.checksum->compute(
          session(),
          m_dumper->get_query_comment(task.name, task.id, "checksumming"));
    }

    m_dumper->checksum_task_finished();
  }

  void write_schema_metadata_impl(const Schema_info &schema) const {
    m_dumper->write_schema_metadata(schema, session());

    ++m_dumper->m_schema_metadata_written;

    m_dumper->maybe_ddl_dump_finished(session());
  }

  void write_table_metadata_impl(const Table_task &table) const {
    m_dumper->write_table_metadata(table, session());

    ++m_dumper->m_table_metadata_written;

    m_dumper->maybe_ddl_dump_finished(session());
  }

  void create_table_data_tasks_impl(const Table_task &table) const {
    auto ranges = create_ranged_tasks(table);

    if (0 == ranges) {
      create_and_push_whole_table_data_task(table);
      ++ranges;
    }

    log_info("%sData dump for table %s will be written to %zu file%s",
             m_log_id.c_str(), table.task_name.c_str(), ranges,
             ranges > 1 ? "s" : "");

    m_dumper->chunking_task_finished();
  }

  template <typename T, std::enable_if_t<std::is_integral_v<T>, int> = 0>
  static std::string quote(const T &v) {
    return std::to_string(v);
  }

  static std::string quote(const Decimal &d) {
    return '\'' + d.to_string() + '\'';
  }

  template <typename T>
  static std::string between(const std::string &column, T begin, T end) {
    assert(begin <= end);

    if (begin == end) {
      return column + "=" + quote(begin);
    } else {
      return column + " BETWEEN " + quote(begin) + " AND " + quote(end);
    }
  }

  struct Chunking_info {
    const Table_task *table;
    uint64_t row_count;
    uint64_t rows_per_chunk;
    uint64_t accuracy;
    std::string partition;
    std::string boundary;
    std::string order_by;
    std::string order_by_desc;
    std::size_t index_column;
  };

  static std::string compare(const Chunking_info &info, const Row &value,
                             const std::string &op, bool eq) {
    assert(info.table->index.info);
    const auto &columns = info.table->index.info->columns();
    const auto size = columns.size() - info.index_column;

    assert(size == value.size());

    auto column = columns.rbegin();
    const auto end = column + size;
    std::size_t idx = size;

    const auto get_value = [&column, &value, &idx]() {
      return quote_value(value[--idx], (*column)->type);
    };

    std::string result = (*column)->quoted_name + op;

    if (eq) {
      result += "=";
    }

    result += get_value();

    while (++column != end) {
      const auto val = get_value();
      result = (*column)->quoted_name + op + val + " OR(" +
               (*column)->quoted_name + "=" + val + " AND(" + result + "))";
    }

    return "(" + result + ")";
  }

  static std::string compare_ge(const Chunking_info &info, const Row &value) {
    return compare(info, value, ">", true);
  }

  static std::string compare_le(const Chunking_info &info, const Row &value) {
    return compare(info, value, "<", true);
  }

  static std::string ge(const Chunking_info &info, const Row &value) {
    std::string result = info.boundary;

    if (!result.empty()) {
      result += " AND";
    }

    result += compare_ge(info, value);

    return result;
  }

  static std::string between(const Chunking_info &info, const Row &begin,
                             const Row &end) {
    return ge(info, begin) + "AND" + compare_le(info, end);
  }

  template <typename T>
  static std::string between(const Chunking_info &info, T begin, T end) {
    std::string result = info.boundary;

    if (!result.empty()) {
      result += " AND ";
    }

    return result + between(info.table->index.info->columns()[info.index_column]
                                ->quoted_name,
                            begin, end);
  }

  static std::string where(const Table_task &task,
                           const std::string &boundary) {
    std::string result;

    if (!task.extra_filter.empty() || !boundary.empty()) {
      // " WHERE " + extra_filter + " AND " + boundary
      result.reserve(12 + task.extra_filter.length() + boundary.length());
      result = " WHERE ";

      if (!task.extra_filter.empty()) {
        result += task.extra_filter;
      }

      if (!task.extra_filter.empty() && !boundary.empty()) {
        result += " AND ";
      }

      if (!boundary.empty()) {
        result += boundary;
      }
    }

    return result;
  }

  static inline std::string where(const Table_data_task &task) {
    return where(task, task.boundary);
  }

  static inline std::string where(const Chunking_info &info) {
    return where(*info.table, info.boundary);
  }

  template <typename T, std::enable_if_t<std::is_integral_v<T>, int> = 0>
  static uint64_t distance(const T &min, const T &max) {
    // it should be (max - min + 1), but this can potentially overflow, check
    // for overflow first
    uint64_t result = max - min;

    if (result < std::numeric_limits<uint64_t>::max()) {
      ++result;
    }

    return result;
  }

  static Decimal distance(const Decimal &min, const Decimal &max) {
    return max - min + 1;
  }

  template <typename T>
  static T ensure_not_zero(T &&v) {
    T t = std::forward<T>(v);

    if (0 == t) {
      ++t;
    }

    return t;
  }

  template <typename T, typename U>
  static T cast(const U &value) {
    // if estimated_step is greater than maximum possible value, casting will
    // overflow, use max instead
    return value < static_cast<U>(std::numeric_limits<T>::max())
               ? static_cast<T>(value)
               : std::numeric_limits<T>::max();
  }

  template <typename T>
  static T sum(const T &value, const T &addend) {
    // if sum is greater than max, use max instead
    return std::numeric_limits<T>::max() - addend <= value
               ? std::numeric_limits<T>::max()
               : value + addend;
  }

  template <typename T>
  static T constant_step(const T & /* from */, const T &step) {
    return step;
  }

  template <typename T>
  T adaptive_step(const T &from, const T &step, const T &max,
                  const Chunking_info &info,
                  const std::string &chunk_id) const {
    static constexpr int k_chunker_retries = 10;
    static constexpr int k_chunker_iterations = 20;

    const auto double_step = 2 * step;
    auto middle = from;

    auto rows = info.rows_per_chunk;
    const auto comment = this->get_query_comment(*info.table, chunk_id);

    int retry = 0;
    uint64_t delta = info.accuracy + 1;

    const auto row_count = [&info, &comment, this](const auto begin,
                                                   const auto end) {
      return row_count_from_explain(
          query("EXPLAIN FORMAT=JSON SELECT " +
                m_dumper->optimizer_hints(info.table->info) + "COUNT(*) FROM " +
                info.table->quoted_name + info.partition +
                where(*info.table, between(info, begin, end)) + info.order_by +
                comment)
              ->fetch_one_or_throw()
              ->get_as_string(0));
    };

    while (delta > info.accuracy && retry < k_chunker_retries) {
      if (max - retry * double_step <= from) {
        // if left boundary is greater than max, stop here
        middle = max;
        break;
      }

      // each time search in a different range, we didn't find the answer in the
      // previous one
      auto left = from + retry * double_step;
      auto right = sum(left, double_step);

      assert(left < right);

      for (int i = 0; i < k_chunker_iterations; ++i) {
        middle = left + (right - left) / 2;

        if (middle >= right || middle <= left) {
          break;
        }

        rows = row_count(from, middle);

        if (0 == i && rows < info.rows_per_chunk) {
          // if in the first iteration there's not enough rows, check the whole
          // range, if there's still not enough rows we can skip this range
          const auto total_rows = row_count(from, right);

          if (total_rows < info.rows_per_chunk) {
            middle = right;
            delta = info.rows_per_chunk - total_rows;
            break;
          }
        }

        if (rows > info.rows_per_chunk) {
          right = middle;
          delta = rows - info.rows_per_chunk;
        } else {
          left = middle;
          delta = info.rows_per_chunk - rows;
        }

        if (delta <= info.accuracy) {
          // we're close enough
          break;
        }
      }

      if (delta > info.accuracy) {
        if (rows >= info.rows_per_chunk) {
          // we have too many rows, but that's OK...
          retry = k_chunker_retries;
        } else {
          if (middle >= max) {
            // we've reached the upper boundary, stop here
            retry = k_chunker_retries;
          } else {
            // we didn't find enough rows here, move farther to
            // the right
            ++retry;
          }
        }
      }
    }

    return ensure_not_zero(middle - from);
  }

  template <typename T>
  std::size_t chunk_integer_column(const Chunking_info &info, const T &min,
                                   const T &max) const {
    std::size_t ranges_count = 0;

    // if rows_per_chunk <= 1 it may mean that the rows are bigger than chunk
    // size, which means we # chunks ~= # rows
    const auto estimated_chunks =
        info.rows_per_chunk > 0
            ? std::max(info.row_count / info.rows_per_chunk, UINT64_C(1))
            : info.row_count;

    using step_t = std::remove_cvref_t<decltype(min)>;
    const auto index_range = distance(min, max);
    const auto row_count_accuracy = std::max(info.row_count / 10, UINT64_C(1));
    const auto estimated_step =
        cast<step_t>(ensure_not_zero(index_range / estimated_chunks));
    // use constant step if number of chunks is small or index range is close to
    // the number of rows
    const bool use_constant_step =
        estimated_chunks < 2 ||
        (index_range > info.row_count
             ? index_range - info.row_count
             : info.row_count - index_range) <= row_count_accuracy;

    std::string chunk_id;
    const auto next_step =
        use_constant_step
            ? std::function<step_t(const step_t &, const step_t &)>(
                  constant_step<T>)
            // using the default capture [&] below results in problems with
            // GCC 5.4.0 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80543)
            : [&info, &max, &chunk_id, this](const auto &from,
                                             const auto &step) {
                return this->adaptive_step(from, step, max, info, chunk_id);
              };

    auto current = min;
    const auto step = estimated_step;

    log_info("%sChunking %s using integer algorithm with %s step",
             m_log_id.c_str(), info.table->task_name.c_str(),
             use_constant_step ? "constant" : "adaptive");

    bool last_chunk = false;

    while (!last_chunk) {
      if (m_dumper->m_worker_interrupt.test()) {
        return ranges_count;
      }

      chunk_id = std::to_string(ranges_count);
      const auto begin = current;
      auto new_step = next_step(current, step);

      // ensure that there's no integer overflow
      --new_step;
      current = (current > max - new_step ? max : current + new_step);

      const auto end = current;

      last_chunk = (current >= max);

      create_and_push_table_data_chunk_task(*info.table,
                                            between(info, begin, end), chunk_id,
                                            ranges_count++, last_chunk);

      ++current;
    }

    return ranges_count;
  }

  std::size_t chunk_integer_column(const Chunking_info &info, const Row &begin,
                                   const Row &end) const {
    log_info("%sChunking %s using integer algorithm", m_log_id.c_str(),
             info.table->task_name.c_str());

    assert(info.table->index.info);
    const auto type =
        info.table->index.info->columns()[info.index_column]->type;

    if (mysqlshdk::db::Type::Integer == type) {
      return chunk_integer_column(info, to_int64_t(begin[info.index_column]),
                                  to_int64_t(end[info.index_column]));
    } else if (mysqlshdk::db::Type::UInteger == type) {
      return chunk_integer_column(info, to_uint64_t(begin[info.index_column]),
                                  to_uint64_t(end[info.index_column]));
    } else if (mysqlshdk::db::Type::Decimal == type) {
      return chunk_integer_column(info, Decimal{begin[info.index_column]},
                                  Decimal{end[info.index_column]});
    }

    throw std::logic_error(
        "Unsupported column type for the integer algorithm: " +
        mysqlshdk::db::to_string(type));
  }

  std::size_t chunk_non_integer_column(const Chunking_info &info,
                                       const Row &begin, const Row &end) const {
    log_info("%sChunking %s using non-integer algorithm", m_log_id.c_str(),
             info.table->task_name.c_str());

    std::size_t ranges_count = 0;

    Row range_begin = begin;
    Row range_end;

    assert(info.table->index.info);
    const auto &columns = info.table->index.info->columns();
    const auto index =
        shcore::str_join(columns.begin() + info.index_column, columns.end(),
                         ",", [](const auto &c) { return c->quoted_name; });

    const auto select =
        "SELECT " + m_dumper->optimizer_hints(info.table->info) + index +
        " FROM " + info.table->quoted_name + info.partition + " ";
    const auto order_by_and_limit = info.order_by + " LIMIT " +
                                    std::to_string(info.rows_per_chunk - 1) +
                                    ",2 ";

    const auto fetch =
        [&end](const std::shared_ptr<mysqlshdk::db::IResult> &res) {
          const auto row = res->fetch_one();
          return row ? fetch_row(row) : end;
        };

    std::shared_ptr<mysqlshdk::db::IResult> result;

    do {
      const auto condition = where(*info.table, ge(info, range_begin));
      const auto chunk_id = std::to_string(ranges_count);
      const auto comment = get_query_comment(*info.table, chunk_id);

      result = query(select + condition + order_by_and_limit + comment);

      if (m_dumper->m_worker_interrupt.test()) {
        return 0;
      }

      range_end = fetch(result);

      create_and_push_table_data_chunk_task(
          *info.table, between(info, range_begin, range_end), chunk_id,
          ranges_count++, range_end == end);

      range_begin = fetch(result);
    } while (range_end != end);

    return ranges_count;
  }

  std::size_t chunk_column(const Chunking_info &info) const {
    if (!info.table->index.info) {
      log_info(
          "%sTable %s does not have a valid index, number of chunks is "
          "estimated",
          m_log_id.c_str(), info.table->task_name.c_str());

      create_and_push_whole_table_data_in_chunks_task(*info.table);
      // we add one because we create an empty final chunk, plus we don't want
      // to return zero here
      return info.row_count / info.rows_per_chunk + 1;
    }

    const auto sql = "SELECT " + m_dumper->optimizer_hints(info.table->info) +
                     info.table->index.info->columns_sql() + " FROM " +
                     info.table->quoted_name + info.partition + where(info);

    auto result = query(sql + info.order_by + " LIMIT 1");
    auto row = result->fetch_one();

    const auto handle_empty_table = [&info, this]() {
      create_and_push_table_data_chunk_task(*info.table, info.boundary, "0", 0,
                                            true);
      return 1;
    };

    if (!row) {
      return handle_empty_table();
    }

    const Row begin = fetch_row(row);

    result = query(sql + info.order_by_desc + " LIMIT 1");
    row = result->fetch_one();

    if (!row) {
      return handle_empty_table();
    }

    const Row end = fetch_row(row);

    const auto type =
        info.table->index.info->columns()[info.index_column]->type;

    log_info("%sChunking %s, using columns: %s, min: [%s], max: [%s]",
             m_log_id.c_str(), info.table->task_name.c_str(),
             shcore::str_join(info.table->index.info->columns(), ", ",
                              [](const auto &c) {
                                return c->quoted_name + " (" +
                                       mysqlshdk::db::to_string(c->type) + ")";
                              })
                 .c_str(),
             shcore::str_join(begin, ", ").c_str(),
             shcore::str_join(end, ", ").c_str());

    if (mysqlshdk::db::Type::Integer == type ||
        mysqlshdk::db::Type::UInteger == type ||
        mysqlshdk::db::Type::Decimal == type) {
      return chunk_integer_column(info, begin, end);
    } else {
      return chunk_non_integer_column(info, begin, end);
    }
  }

  std::size_t create_ranged_tasks(const Table_task &table) const {
    if (!m_dumper->m_options.split()) {
      return 0;
    }

    mysqlshdk::utils::Duration duration;
    duration.start();

    const auto &task_name = table.task_name;

    // default row size to use when there's no known row size
    constexpr const uint64_t k_default_row_size = 256;

    assert(table.partitions.size() < 2);
    const auto partition =
        table.partitions.empty() ? nullptr : table.partitions[0].info;

    auto average_row_length = partition ? partition->average_row_length
                                        : table.info->average_row_length;
    auto partition_clause =
        partition ? " PARTITION (" + partition->quoted_name + ")" : "";

    if (0 == average_row_length) {
      average_row_length = k_default_row_size;

      const auto result = query("SELECT 1 FROM " + table.quoted_name +
                                partition_clause + " LIMIT 1");

      // print the message only if table is not empty
      if (result->fetch_one()) {
        auto msg = "Table statistics not available for " + task_name +
                   ", chunking operation may be not optimal. "
                   "Please consider running 'ANALYZE TABLE " +
                   table.quoted_name + ";'";

        if (partition) {
          msg += " or 'ALTER TABLE " + table.quoted_name +
                 " ANALYZE PARTITION " + partition->quoted_name + ";'";
        }

        msg += " first.";

        current_console()->print_note(msg);
      }
    }

    Chunking_info info;

    info.table = &table;
    info.row_count = partition ? partition->row_count : table.info->row_count;
    info.rows_per_chunk =
        m_dumper->m_options.bytes_per_chunk() / average_row_length;
    info.accuracy = std::max(info.rows_per_chunk / 10, UINT64_C(10));
    info.partition = std::move(partition_clause);

    if (table.index.info) {
      if (!table.index.is_pke) {
        for (const auto &c : table.index.info->columns()) {
          if (c->nullable) {
            if (!info.boundary.empty()) {
              info.boundary += "AND";
            }

            info.boundary += "(" + c->quoted_name + " IS NOT NULL)";
          }
        }
      }

      info.order_by = " ORDER BY " + table.index.info->columns_sql();
      info.order_by_desc =
          " ORDER BY " +
          shcore::str_join(table.index.info->columns(), ",", [](const auto &c) {
            return c->quoted_name + " DESC";
          });
      info.index_column = 0;
    }

    log_info("%sChunking %s, rows: %" PRIu64 ", average row length: %" PRIu64
             ", rows per chunk: %" PRIu64,
             m_log_id.c_str(), task_name.c_str(), info.row_count,
             average_row_length, info.rows_per_chunk);

    const auto ranges_count = chunk_column(info);

    duration.finish();
    log_debug("%sChunking of %s took %f seconds", m_log_id.c_str(),
              task_name.c_str(), duration.seconds_elapsed());

    return ranges_count;
  }

  void handle_exception(const std::string &context, const char *msg) {
    m_dumper->m_worker_exceptions[m_id] = std::current_exception();
    m_dumper->m_worker_exception_thrown = true;
    current_console()->print_error(
        (context.empty() ? "" : m_log_id + "Error while " + context + ": ") +
        msg);

    if (Exception_strategy::ABORT == m_strategy) {
      m_dumper->emergency_shutdown();
    }
  }

  void dump_schema_ddl_impl(const Schema_info &schema) const {
    const auto dumper = m_dumper->schema_dumper(session());

    if (m_dumper->uses_multifile_schema_ddl()) {
      m_dumper->write_ddl(*m_dumper->dump_schema(dumper.get(), schema.name),
                          common::get_schema_filename(schema.basename));

      if (m_dumper->m_options.dump_events()) {
        m_dumper->write_ddl(
            *m_dumper->dump_events(dumper.get(), schema.name),
            common::get_schema_filename(schema.basename, "events.sql"));
      }

      if (m_dumper->m_options.dump_libraries()) {
        m_dumper->write_ddl(
            *m_dumper->dump_libraries(dumper.get(), schema.name),
            common::get_schema_filename(schema.basename, "libraries.sql"));
      }

      if (m_dumper->m_options.dump_routines()) {
        m_dumper->write_ddl(
            *m_dumper->dump_routines(dumper.get(), schema.name),
            common::get_schema_filename(schema.basename, "routines.sql"));
      }
    } else {
      m_dumper->write_ddl(
          *m_dumper->dump_complete_schema(dumper.get(), schema.name),
          common::get_schema_filename(schema.basename));
    }

    ++m_dumper->m_ddl_written;

    m_dumper->maybe_ddl_dump_finished(session());
  }

  void dump_table_ddl_impl(const Schema_info &schema,
                           const Table_info &table) const {
    const auto dumper = m_dumper->schema_dumper(session());

    m_dumper->write_ddl(
        *m_dumper->dump_table(dumper.get(), schema.name, table.name),
        common::get_table_filename(table.basename));

    if (m_dumper->m_options.dump_triggers() &&
        dumper->count_triggers_for_table(schema.name, table.name) > 0) {
      m_dumper->write_ddl(
          *m_dumper->dump_triggers(dumper.get(), schema.name, table.name),
          common::get_table_data_filename(table.basename, "triggers.sql"));
    }

    ++m_dumper->m_ddl_written;

    m_dumper->maybe_ddl_dump_finished(session());
  }

  void dump_view_ddl_impl(const Schema_info &schema,
                          const View_info &view) const {
    const auto dumper = m_dumper->schema_dumper(session());

    // DDL file with the temporary table
    m_dumper->write_ddl(
        *m_dumper->dump_temporary_view(dumper.get(), schema.name, view.name),
        common::get_table_data_filename(view.basename, "pre.sql"));

    // DDL file with the view structure
    m_dumper->write_ddl(
        *m_dumper->dump_view(dumper.get(), schema.name, view.name),
        common::get_table_filename(view.basename));

    ++m_dumper->m_ddl_written;

    m_dumper->maybe_ddl_dump_finished(session());
  }

  std::string get_query_comment(const Table_task &table,
                                const std::string &id) const {
    return m_dumper->get_query_comment(table.task_name, id, "chunking");
  }

  uint64_t row_count_from_explain(const std::string &explain) const {
    const auto error = [&explain](const std::string &msg) {
      log_error("JSON output of malformed EXPLAIN statement:\n%s",
                explain.c_str());
      return std::runtime_error(msg);
    };

    shcore::json::JSON json;

    try {
      json = shcore::json::parse_object_or_throw(explain);
    } catch (const std::exception &e) {
      throw error(shcore::str_format(
          "Failed to parse JSON output of an EXPLAIN statement: %s", e.what()));
    }

    if (!m_json_path) {
      for (const auto path : {
               "/query_plan/inputs/0/estimated_rows",  // v2 + BUG#35239659
               "/inputs/0/estimated_rows",             // JSON format ver. 2
               "/query_block/table/rows_examined_per_scan",   // 5.7+ (ver. 1)
               "/query_block/ordering_operation/table/rows",  // 5.6
               "/query_block/nested_loop/0/table/rows",       // MariaDB
           }) {
        if (rapidjson::Pointer(path).Get(json)) {
          m_json_path = path;
          break;
        }
      }

      if (!m_json_path) {
        throw error(
            "Couldn't find the row count in JSON output of an EXPLAIN "
            "statement");
      }
    }

    const auto value = rapidjson::Pointer(m_json_path).Get(json);

    if (!value) {
      throw error(
          shcore::str_format("Couldn't find the row count in JSON output of an "
                             "EXPLAIN statement using path: %s",
                             m_json_path));
    }

    if (value->IsNumber()) {
      if (value->IsDouble()) {
        return static_cast<uint64_t>(value->GetDouble());
      } else {
        return value->GetUint64();
      }
    } else {
      throw error(
          "The row count in JSON output of an EXPLAIN statement is not a "
          "number");
    }
  }

  const std::size_t m_id;
  const std::string m_log_id;
  Dumper *m_dumper;
  Exception_strategy m_strategy;
  std::shared_ptr<mysqlshdk::db::ISession> m_session;
  // JSON path in an output of EXPLAIN statement to the row count
  mutable const char *m_json_path = nullptr;
};

// template specialization of a static method must be defined outside of a class
template <>
Decimal Dumper::Table_worker::cast(const Decimal &value) {
  return value;
}

template <>
Decimal Dumper::Table_worker::sum(const Decimal &value, const Decimal &delta) {
  return value + delta;
}

Dumper::Dumper(const Dump_options &options)
    : m_options(options),
      m_basenames(k_max_basename_length),
      m_progress_thread("Dump", options.show_progress()) {
  try {
    initialize_dumper();
  } catch (...) {
    log_exception("dumper's initialization", std::current_exception());
    throw;
  }
}

void Dumper::initialize_dumper() {
  if (m_options.use_single_file()) {
    m_output_file =
        m_options.make_file(m_options.output_url(), m_options.compression(),
                            m_options.compression_options());

    if (m_output_file->is_local()) {
      const auto parent = m_output_file->parent();

      if (!parent->exists()) {
        throw std::invalid_argument(
            "Cannot proceed with the dump, the directory containing '" +
            m_options.output_url() +
            "' does not exist at the target location '" +
            parent->full_path().masked() + "'.");
      }
    } else {
      // verify if file is writeable
      m_output_file->open(mysqlshdk::storage::Mode::WRITE);
      m_output_file->close();
    }
  } else {
    m_dump_config.dir = m_options.make_directory(m_options.output_url());

    if (dump_dir()->exists()) {
      if (!dump_dir()->is_empty()) {
        const auto files =
            mysqlshdk::storage::sort(dump_dir()->list(true).files);

        std::vector<std::string> file_data;
        const auto full_path = dump_dir()->full_path();

        for (const auto &file : files) {
          file_data.push_back(shcore::str_format(
              "%s [size %zu]",
              dump_dir()->join_path(full_path.masked(), file.name()).c_str(),
              file.size()));
        }

        log_error(
            "Unable to dump to %s, the directory exists and is not empty:\n  "
            "%s",
            full_path.masked().c_str(),
            shcore::str_join(file_data, "\n  ").c_str());

        const auto config = m_options.storage_config(m_options.output_url());
        const auto masked_url =
            mysqlshdk::oci::mask_any_par(m_options.output_url());

        if (config && config->valid()) {
          throw std::invalid_argument(
              "Cannot proceed with the dump, " + config->description() +
              " already contains files with the specified prefix '" +
              masked_url + "'.");
        } else {
          throw std::invalid_argument(
              "Cannot proceed with the dump, the specified directory '" +
              masked_url + "' already exists at the target location " +
              full_path.masked() + " and is not empty.");
        }
      }
    }
  }

  m_dump_config.init(m_options.dialect(), m_options.compression(),
                     m_options.compression_options());

  if (m_options.compatibility_options().is_set(
          Compatibility_option::STRIP_DEFINERS) &&
      compatibility::supports_set_any_definer_privilege(
          m_options.target_version())) {
    current_console()->print_note(
        "The 'targetVersion' option is set to " +
        m_options.target_version().get_base() +
        ". This version supports the SET_ANY_DEFINER privilege, using the "
        "'strip_definers' compatibility option is unnecessary.");
  }

  if (m_options.checksum()) {
    m_checksum = std::make_unique<common::Checksums>();
  }

  m_mmap_options = mysqlshdk::storage::mmap_options();
}

void Dumper::run() {
  if (m_options.is_dry_run()) {
    current_console()->print_note(
        "dryRun enabled, no locks will be acquired and no files will be "
        "created.");
  }

  try {
    do_run();
  } catch (...) {
    kill_workers();
    translate_current_exception(shcore::str_format("%s()", name()),
                                m_progress_thread);
  }

  if (m_worker_interrupt.test()) {
    // m_worker_interrupt is also used to signal exceptions from workers,
    // if we're here, then no exceptions were thrown and user pressed ^C
    throw shcore::cancelled("Interrupted by user");
  }
}

void Dumper::interrupt() { m_worker_interrupt.test_and_set(); }

void Dumper::interruption_notification() {
  // this method may be called from another thread, make sure we're using a
  // console that is able to synchronize with the progress that's being
  // displayed
  m_progress_thread.console()->print_warning(
      "Interrupted by user. Canceling...");
}

void Dumper::abort() {
  emergency_shutdown();
  mysqlshdk::db::kill_query(m_session);
}

void Dumper::do_run() {
  // helper:
  // lock_instance():
  //   1. if LIFB is available to the user
  //     1.1. LOCK INSTANCE FOR BACKUP
  //   2. if LIFB is not available to the user
  //     2.1. if user was able to execute FTWRL:
  //       2.1.1. continue
  //     2.2. if user was not able to execute FTWRL:
  //       2.2.1. if (gtid_mode is ON or ON_PERMISSIVE) or (log_bin is ON):
  //         2.2.1.1. continue
  //       2.2.2. else
  //         2.2.2.1. error and abort
  // start up sequence (consistent dump):
  // 1. acquire read locks:
  //   1.1. if user is able to execute FTWRL:
  //     1.1.1. FLUSH TABLES WITH READ LOCK
  //   1.2. if user is not able to execute FTWRL:
  //     1.2.1. execute lock_instance()
  //     1.2.2. if LIFB is not available to the user, create a new session and
  //            LOCK mysql TABLES using that session
  //     1.2.3. fetch schemas, tables and views to be dumped
  //     1.2.4. create one or more new sessions and LOCK dumped TABLES
  //     1.2.5. if there are any errors, abort
  // 2. start a transaction
  // 3. create worker threads, each thread:
  //   3.1. creates a new session
  //   3.2. starts a transaction
  // 4. gather information about objects to be dumped:
  //   4.1. if 1.2.3. was not executed, do this now
  //   4.2. fetch server metadata (i.e. gtid_executed, binlog file and position)
  //   4.3. fetch information about tables, views, routines, users, etc.
  // 5. wait for all worker threads to finish their initialization
  // 6. execute lock_instance()
  // 7. release read locks:
  //   7.1. if user was able to execute FTWRL:
  //     7.1.1 UNLOCK TABLES
  //   7.2. if user was not able to execute FTWRL:
  //     7.2.1 close all sessions which executed LOCK TABLES
  // 8. workers start to execute their tasks
  // 9. once all DDL-related files are written (metadata, SQL, etc.):
  //   9.1. if both FTWRL and LIFB were not available to the user:
  //     9.1.1. if GTID: fetch gtid_executed, if different from the 4.2. value:
  //       9.1.1.1. if dumpInstance(): error and abort
  //       9.1.1.2. else: warning and continue
  //     9.1.2. else SHOW MASTER STATUS, if different from the 4.2. value:
  //       9.1.2.1. if dumpInstance(): error and abort
  //       9.1.2.2. else: warning and continue

  shcore::on_leave_scope terminate_session([this]() { close_session(); });

  {
    m_worker_interrupt.clear();
    m_progress_thread.start();
    shcore::on_leave_scope cleanup_progress([this]() { shutdown_progress(); });
    const auto init_stage = m_progress_thread.start_stage("Initializing");

    open_session();

    throw_if_cannot_dump_users();

    fetch_user_privileges();

    {
      shcore::on_leave_scope read_locks([this]() { release_read_locks(); });

      acquire_read_locks();

      if (m_worker_interrupt.test()) {
        return;
      }

      create_worker_sessions();

      create_worker_threads();

      init_stage->finish();

      // initialize cache while threads are starting up
      initialize_instance_cache();

      if (m_options.consistent_dump() && !m_worker_interrupt.test()) {
        current_console()->print_info("All transactions have been started");
        lock_instance();
      }

      if (!m_worker_interrupt.test() && !m_options.is_export_only() &&
          is_gtid_executed_inconsistent()) {
        current_console()->print_warning(
            "The dumped value of gtid_executed is not guaranteed to be "
            "consistent");
      }
    }

    shcore::on_leave_scope cleanup([this]() {
      // Ensures all the worker sessions get closed
      size_t worker_sessions = m_options.threads();
      while (worker_sessions) {
        auto session = m_session_pool.pop();

        if (!m_worker_exception_thrown) {
          assert_transaction_is_open(session);
        }
        session->close();
        --worker_sessions;
      }
    });

    create_schema_tasks();

    validate_privileges();
    initialize_counters();
    validate_mds();
    validate_data_masking();

    initialize_dump();

    dump_ddl();

    create_schema_metadata_tasks();
    create_schema_ddl_tasks();
    create_table_tasks();

    if (!m_worker_interrupt.test()) {
      std::string msg;

      if (!m_options.is_dry_run()) {
        msg = "Running data dump using " + std::to_string(m_options.threads()) +
              " thread";

        if (m_options.threads() > 1) {
          msg += 's';
        }

        msg += '.';
      }

      if (m_checksum) {
        if (!msg.empty()) {
          msg += ' ';
        }

        msg += "Checksumming enabled.";
      }

      if (!msg.empty()) {
        current_console()->print_status(msg);
      }

      if (!m_options.is_dry_run() && m_options.show_progress() &&
          m_options.dump_data()) {
        current_console()->print_note(
            "Progress information uses estimated values and may not be "
            "accurate.");
      }
    }

    initialize_throughput_progress();
    initialize_checksum_progress();

    maybe_push_shutdown_tasks();
    wait_for_all_tasks();

    if (!m_worker_interrupt.test()) {
      finalize_dump();
    }

    rethrow();
  }

  if (!m_options.is_dry_run() && !m_worker_interrupt.test()) {
    summarize();
  }

#ifndef NDEBUG
  if (m_server_version.number < Version(8, 0, 21) ||
      m_server_version.number > Version(8, 0, 23) || !dump_users()) {
    // SHOW CREATE USER auto-commits transaction in some 8.0 versions, we don't
    // check if transaction is still open in such case if users were dumped
    assert_transaction_is_open(session());
  }
#endif  // !NDEBUG
}

const std::shared_ptr<mysqlshdk::db::ISession> &Dumper::session()
    const noexcept {
  assert(m_session);
  // ensure that the main session is only used from the main thread
  assert(std::this_thread::get_id() == m_main_thread);
  return m_session;
}

std::unique_ptr<Schema_dumper> Dumper::schema_dumper(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  auto dumper = std::make_unique<Schema_dumper>(session, m_cache);

  dumper->use_filters(&m_options.filters());
  dumper->use_shared_session(m_shared_session);

  dumper->opt_comments = true;
  dumper->opt_drop_database = false;
  dumper->opt_drop_table = false;
  dumper->opt_drop_view = true;
  dumper->opt_drop_event = true;
  dumper->opt_drop_routine = true;
  dumper->opt_drop_library = true;
  dumper->opt_drop_trigger = true;
  dumper->opt_databases = true;
  dumper->opt_reexecutable = true;
  dumper->opt_tz_utc = m_options.use_timezone_utc();
  dumper->opt_mysqlaas = m_options.mds_compatibility();
  dumper->set_target_version(m_options.target_version());
  dumper->opt_report_deprecated_errors_as_warnings =
      m_options.implicit_target_version();
  dumper->opt_character_set_results = m_options.character_set();
  dumper->opt_column_statistics = false;

  return dumper;
}

void Dumper::on_init_thread_session(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  // transaction cannot be started here, as the main thread has to acquire read
  // locks first
  execute(session, "SET SQL_MODE = '';");
  executef(session, "SET NAMES ?;", m_options.character_set());

  // The amount of time the server should wait for us to read data from it
  // like resultsets. Result reading can be delayed by slow uploads.
  executef(session, "SET SESSION net_write_timeout = ?",
           k_mysql_server_net_write_timeout);

  // Amount of time before server disconnects idle clients.
  executef(session, "SET SESSION wait_timeout = ?",
           k_mysql_server_wait_timeout);

  if (m_options.use_timezone_utc()) {
    execute(session, "SET TIME_ZONE = '+00:00';");
  }

  execute("SET SQL_QUOTE_SHOW_CREATE=1");
}

void Dumper::open_session() {
  using shcore::option_tracker::Shell_feature;

#ifndef NDEBUG
  m_main_thread = std::this_thread::get_id();
#endif  // !NDEBUG
  Shell_feature feature_id = Shell_feature::NONE;
  if (m_options.report_dump_option()) {
    feature_id = Shell_feature::UTIL_DUMP;
  }
  // Establish session and configure it.
  m_session = establish_session(m_options.connection_options(), false, false,
                                true, feature_id);
  on_init_thread_session(session());

  fetch_server_information();

  log_server_version();

  if (m_checksum) {
    m_checksum->configure(session());
  }

  // shared session used by the Schema_dumper
  m_shared_session = mysqlshdk::db::make_shared_session(
      establish_session(m_options.connection_options(), false));
  on_init_thread_session(m_shared_session->acquire());
}

void Dumper::close_session() {
  if (m_session) {
    m_session->close();
  }
}

void Dumper::fetch_user_privileges() {
  using mysqlshdk::mysql::Instance;
  using mysqlshdk::mysql::User_privileges;

  const auto instance = Instance(session());

  instance.get_current_user(&m_user_account.user, &m_user_account.host);

  m_user_privileges = instance.get_user_privileges(m_user_account.user,
                                                   m_user_account.host, true);
  m_skip_grant_tables_active = "'skip-grants user'@'skip-grants host'" ==
                               shcore::make_account(m_user_account);

  if (m_server_version.number >= Version(8, 0, 0)) {
    m_user_has_backup_admin =
        !m_user_privileges->validate({"BACKUP_ADMIN"}).has_missing_privileges();
  }

  warn_about_backup_lock();
}

void Dumper::warn_about_backup_lock() const {
  if (!m_options.consistent_dump()) {
    return;
  }

  if (!m_user_has_backup_admin) {
    current_console()->print_note(
        "Backup lock is not " + why_backup_lock_is_missing() +
        " and DDL changes will not be blocked. The dump may fail with an error "
        "if schema changes are made while dumping.");
  }
}

std::string Dumper::why_backup_lock_is_missing() const {
  return m_server_version.number >= Version(8, 0, 0)
             ? "available to the account " +
                   shcore::make_account(m_user_account)
             : "supported in MySQL " + m_server_version.number.get_short();
}

void Dumper::lock_all_tables() {
  lock_instance();

  // find out the max query size we can send
  uint64_t max_packet_size = 0;

  {
    const auto r = query("select @@max_allowed_packet");
    // 1024 is the minimum allowed value
    max_packet_size = r->fetch_one_or_throw()->get_uint(0, 1024);
  }

  const auto console = current_console();
  const std::string k_lock_tables = "LOCK TABLES ";

  using mysqlshdk::mysql::User_privileges;
  const std::set<std::string> privileges = {"LOCK TABLES", "SELECT"};

  const auto validate_privileges = [this, &privileges](
                                       const std::string &schema,
                                       const std::string &table =
                                           User_privileges::k_wildcard) {
    const auto result = m_user_privileges->validate(privileges, schema, table);

    if (result.has_missing_privileges()) {
      const std::string object =
          User_privileges::k_wildcard == table
              ? "schema " + shcore::quote_identifier(schema)
              : "table " + quote(schema, table);

      THROW_ERROR(SHERR_DUMP_LOCK_TABLES_MISSING_PRIVILEGES,
                  shcore::make_account(m_user_account).c_str(), object.c_str(),
                  shcore::str_join(result.missing_privileges(), ", ").c_str());
    }
  };

  const auto lock_tables = [this](std::string *lock_statement) {
    if (',' == lock_statement->back()) {
      // remove last comma
      lock_statement->pop_back();
    }

    log_info("Locking tables: %s", lock_statement->c_str());

    if (!m_options.is_dry_run()) {
      // initialize session
      auto s = establish_session(session()->get_connection_options(), false);
      on_init_thread_session(s);
      execute(s, *lock_statement);
      m_lock_sessions.emplace_back(std::move(s));
    }
  };

  if (m_user_has_backup_admin) {
    current_console()->print_note(
        "Instance locked for backup, skipping mysql system tables locks");
  } else {
    const auto non_fatal_failure = [&console](const std::string &error) {
      console->print_warning("Could not lock mysql system tables: " + error);
      console->print_warning(
          "The dump will continue, but the dump may not be completely "
          "consistent if changes to accounts or routines are made during it.");
    };

    // lock relevant tables in mysql so that grants, views, routines etc. are
    // consistent too, use the main thread to lock these tables, as it is going
    // to read from them
    try {
      const std::string mysql = "mysql";

      // if user doesn't have the SELECT privilege on mysql.*, it's not going to
      // be possible to list tables
      validate_privileges(mysql);

      const auto result = query(
          "SHOW TABLES IN mysql WHERE Tables_in_mysql IN"
          "('columns_priv', 'db', 'default_roles', 'func', 'global_grants', "
          "'proc', 'procs_priv', 'proxies_priv', 'role_edges', 'tables_priv', "
          "'user')");

      auto stmt = k_lock_tables;

      while (auto row = result->fetch_one()) {
        const auto table = row->get_string(0);

        validate_privileges(mysql, table);

        stmt.append("mysql." + shcore::quote_identifier(table) + " READ,");
      }

      lock_tables(&stmt);
    } catch (const mysqlshdk::db::Error &e) {
      if (ER_SPECIFIC_ACCESS_DENIED_ERROR == e.code() ||
          ER_DBACCESS_DENIED_ERROR == e.code() ||
          ER_ACCESS_DENIED_ERROR == e.code()) {
        non_fatal_failure(e.format());
      } else {
        console->print_error("Could not lock mysql system tables: " +
                             e.format());
        throw;
      }
    } catch (const std::runtime_error &e) {
      non_fatal_failure(e.what());
    }
  }

  initialize_instance_cache_minimal();

  // get tables to be locked
  std::vector<std::string> tables;

  for (const auto &schema : m_cache.schemas) {
    for (const auto &table : schema.second.tables) {
      validate_privileges(schema.first, table.first);

      tables.emplace_back(quote(schema.first, table.first) + " READ,");
    }
  }

  // iterate all tables that are going to be dumped and LOCK TABLES them, use
  // separate sessions to run the queries as executing LOCK TABLES implicitly
  // unlocks tables locked before
  try {
    const auto k_lock_tables_size = k_lock_tables.size();
    auto size = std::string::npos;
    auto stmt = k_lock_tables;

    for (const auto &table : tables) {
      size = stmt.size();

      // check if we're overflowing the SQL packet (256B slack is probably
      // enough)
      if (size + table.size() >= max_packet_size - 256 &&
          size > k_lock_tables_size) {
        lock_tables(&stmt);

        stmt = k_lock_tables;
      }

      stmt.append(table);
    }

    if (stmt.size() > k_lock_tables_size) {
      // flush the rest
      lock_tables(&stmt);
    }
  } catch (const mysqlshdk::db::Error &e) {
    console->print_error("Error locking tables: " + e.format());
    throw;
  }
}

void Dumper::acquire_read_locks() {
  if (!m_options.consistent_dump()) {
    return;
  }

  // This will block until lock_wait_timeout if there are any sessions with open
  // transactions/locks.
  current_console()->print_info("Acquiring global read lock");

  // user can execute FLUSH statements if has RELOAD privilege or, starting with
  // 8.0.23, FLUSH_TABLES privilege
  const auto execute_ftwrl =
      !m_user_privileges->validate({"RELOAD"}).has_missing_privileges() ||
      (m_server_version.number >= Version(8, 0, 23) &&
       !m_user_privileges->validate({"FLUSH_TABLES"}).has_missing_privileges());
  m_ftwrl_used = execute_ftwrl;

  if (execute_ftwrl) {
    if (m_options.is_dry_run()) {
      // in case of a dry run we need to attempt to acquire FTWRL, because it
      // may require some other privileges, set the lock_wait_timeout to
      // smallest possible value to avoid waiting for long periods of time
      execute("SET @current_lock_wait_timeout = @@session.lock_wait_timeout");
      execute("SET @@session.lock_wait_timeout = 1");
    }

    bool timeout = false;

    try {
      // no need to flush in case of a dry run, we're only interested if FTWRL
      // succeeds
      if (!m_options.is_dry_run()) {
        // We do first a FLUSH TABLES. If a long update is running, the FLUSH
        // TABLES will wait but will not stall the whole mysqld, and when the
        // long update is done the FLUSH TABLES WITH READ LOCK will start and
        // succeed quickly. So, FLUSH TABLES is to lower the probability of a
        // stage where both shell and most client connections are stalled. Of
        // course, if a second long update starts between the two FLUSHes, we
        // have that bad stall.
        execute("FLUSH NO_WRITE_TO_BINLOG TABLES;");
      }

      execute("FLUSH TABLES WITH READ LOCK;");
    } catch (const mysqlshdk::db::Error &e) {
      if (ER_LOCK_WAIT_TIMEOUT == e.code() && m_options.is_dry_run()) {
        // it's a dry run and FTWRL failed due to a timeout, treat this as a
        // success - user had privileges required to execute FTWRL
        timeout = true;
      } else if (ER_SPECIFIC_ACCESS_DENIED_ERROR == e.code() ||
                 ER_DBACCESS_DENIED_ERROR == e.code() ||
                 ER_ACCESS_DENIED_ERROR == e.code()) {
        // FTWRL failed due to an access denied error, fall back to LOCK TABLES
        m_ftwrl_used = false;
      } else {
        current_console()->print_error("Failed to acquire global read lock: " +
                                       e.format());
        THROW_ERROR(SHERR_DUMP_GLOBAL_READ_LOCK_FAILED);
      }
    }

    if (m_options.is_dry_run()) {
      // restore previous value of lock_wait_timeout
      execute("SET @@session.lock_wait_timeout = @current_lock_wait_timeout");

      if (!timeout && m_ftwrl_used) {
        // we've successfully acquired FTWRL, but it's a dry run and we don't
        // want to interfere with the live server, release it immediately
        execute("UNLOCK TABLES");
      }
    }

    if (m_ftwrl_used) {
      current_console()->print_info("Global read lock acquired");
    }
  }

  if (!m_ftwrl_used) {
    current_console()->print_warning(
        "The current user lacks privileges to acquire a global read lock "
        "using 'FLUSH TABLES WITH READ LOCK'. Falling back to LOCK "
        "TABLES...");

    const auto handle_error = [](const std::string &msg) {
      current_console()->print_error(
          "Unable to acquire global read lock neither table read locks.");

      THROW_ERROR(SHERR_DUMP_LOCK_TABLES_FAILED, msg.c_str());
    };

    // if FTWRL isn't executable by the user, try LOCK TABLES instead
    try {
      lock_all_tables();

      current_console()->print_info("Table locks acquired");
    } catch (const mysqlshdk::db::Error &e) {
      handle_error(e.format());
    } catch (const std::runtime_error &e) {
      handle_error(e.what());
    }
  }

  // if FTWRL was used to lock the tables, it is safe to start a transaction, as
  // it will not release the global read lock; if LOCK TABLES was used instead,
  // tables were locked in separate sessions, so it's safe to start the
  // transaction in the main session
  start_transaction(session());
}

void Dumper::release_read_locks() const {
  if (!m_options.consistent_dump()) {
    return;
  }

  if (!m_ftwrl_used) {
    log_info("Releasing the table locks");

    // close the locking sessions, this will release the table locks
    for (const auto &session : m_lock_sessions) {
      session->close();
    }
  } else {
    // FTWRL has succeeded, transaction is active, UNLOCK TABLES will not
    // automatically commit the transaction
    execute("UNLOCK TABLES;");
  }

  if (!m_worker_interrupt.test()) {
    // we've been interrupted, we still need to release locks, but don't
    // inform the user
    current_console()->print_info("Global read lock has been released");
  }
}

void Dumper::start_transaction(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  if (m_options.consistent_dump() && !m_options.is_dry_run()) {
    execute(session,
            "SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;");
    execute(session, "START TRANSACTION WITH CONSISTENT SNAPSHOT;");
  }
}

void Dumper::assert_transaction_is_open(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
#ifdef NDEBUG
  (void)session;
#else  // !NDEBUG
  if (m_options.consistent_dump() && !m_options.is_dry_run()) {
    try {
      // if there's an active transaction, this will throw
      execute(session, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ");
      // no exception -> transaction is not active
      assert(false);
    } catch (const mysqlshdk::db::Error &e) {
      // make sure correct error is reported
      assert(e.code() == ER_CANT_CHANGE_TX_CHARACTERISTICS);
    } catch (...) {
      // any other exception means that something else went wrong
      assert(false);
    }
  }
#endif
}

void Dumper::lock_instance() {
  if (!m_options.consistent_dump() || m_instance_locked) {
    return;
  }

  if (m_user_has_backup_admin) {
    const auto console = current_console();

    console->print_info("Locking instance for backup");

    if (!m_options.is_dry_run()) {
      try {
        execute("LOCK INSTANCE FOR BACKUP");
      } catch (const shcore::Error &e) {
        console->print_error("Could not acquire the backup lock: " +
                             e.format());

        throw;
      }
    }
  } else if (!m_ftwrl_used) {
    auto can_execute_show_status =
        2 != m_user_privileges->validate({"REPLICATION CLIENT", "SUPER"})
                 .missing_privileges()
                 .size();
    DBUG_EXECUTE_IF("dumper_replication_client_unavailable",
                    { can_execute_show_status = false; });
    const auto can_check_dump_consistency =
        can_execute_show_status || m_gtid_enabled;
    const auto console = current_console();

    if (m_binlog_enabled && can_check_dump_consistency) {
      console->print_note(shcore::str_format(
          "The current user does not have required privileges to execute FLUSH "
          "TABLES WITH READ LOCK, backup lock is not %s and DDL changes cannot "
          "be blocked. The DDL consistency will be checked using the binary "
          "log.",
          why_backup_lock_is_missing().c_str()));
    } else {
      const auto fatal_error = shcore::str_caseeq(name(), "dumpInstance");

      auto msg = shcore::str_format(
          R"(The current user does not have required privileges to execute FLUSH TABLES WITH READ LOCK and:
 * Backup lock is not %s and DDL changes cannot be blocked.)",
          why_backup_lock_is_missing().c_str());

      if (!m_binlog_enabled) {
        msg += shcore::str_format("\n * The binary logging is disabled.");
      }

      if (!can_check_dump_consistency) {
        msg +=
            "\n * The gtid_mode system variable is set to OFF or "
            "OFF_PERMISSIVE.";

        msg += shcore::str_format(
            "\n * The current user does not have required privileges to "
            "execute SHOW %s STATUS.",
            mysqlshdk::mysql::get_binary_logs_keyword(m_server_version.number,
                                                      true));
      }

      console->print_warning(msg);

      msg = "The consistency of the dump cannot be guaranteed.";

      if (fatal_error) {
        console->print_error(msg);
      } else {
        console->print_warning(msg);
      }

      msg = "In order to create a consistent dump, either:";

      if (m_server_version.number >= Version(8, 0, 0)) {
        msg += "\n * Use an account which has the BACKUP_ADMIN privilege.";
      }

      if (!m_binlog_enabled) {
        if (can_check_dump_consistency) {
          msg += "\n * Enable binary logging.";
        } else {
          msg +=
              "\n * Enable binary logging and set the gtid_mode system "
              "variable to ON or ON_PERMISSIVE.";

          msg +=
              "\n * Enable binary logging and use an account which has the "
              "REPLICATION CLIENT or SUPER privileges.";
        }
      } else {
        assert(!can_check_dump_consistency);

        msg += "\n * Set the gtid_mode system variable to ON or ON_PERMISSIVE.";

        msg +=
            "\n * Use an account which has the REPLICATION CLIENT or SUPER "
            "privileges.";
      }

      console->print_note(msg);

      if (fatal_error) {
        THROW_ERROR(SHERR_DUMP_CONSISTENCY_CHECK_FAILED);
      }
    }
  }

  m_instance_locked = true;
}

void Dumper::initialize_instance_cache_minimal() {
  m_cache = Instance_cache_builder(session(), m_options.filters(), {}).build();

  validate_schemas_list();
}

void Dumper::initialize_instance_cache() {
  const auto stage = m_progress_thread.start_stage("Gathering information");
  shcore::on_leave_scope finish_stage([stage]() { stage->finish(); });

  auto builder = Instance_cache_builder(session(), m_options.filters(),
                                        std::move(m_cache));

  builder.metadata(m_options.included_partitions());

  if (dump_users()) {
    builder.users();
  }

  if (m_options.dump_ddl()) {
    if (m_options.dump_events()) {
      builder.events();
    }

    if (m_options.dump_libraries()) {
      builder.libraries();
    }

    if (m_options.dump_routines()) {
      builder.routines();
    }

    if (m_options.dump_triggers()) {
      builder.triggers();
    }
  }

  if (m_options.dump_binlog_info()) {
    builder.binlog_info();
  }

  if (should_fetch_data_masking_policies()) {
    builder.data_masking_policies();
  }

  m_cache = builder.build();
  m_cache_initialized = true;

  validate_schemas_list();

  print_object_stats();
}

void Dumper::create_schema_tasks() {
  bool has_partitions = false;

  for (const auto &s : m_cache.schemas) {
    Schema_info schema;
    schema.name = s.first;
    schema.quoted_name = shcore::quote_identifier(schema.name);
    schema.basename = get_basename(common::encode_schema_basename(schema.name));

    for (const auto &t : s.second.tables) {
      Table_info table;
      table.name = t.first;
      table.quoted_name = quote(schema.name, table.name);
      table.basename =
          get_basename(common::encode_table_basename(schema.name, table.name));
      table.info = &t.second;

      for (const auto &p : t.second.partitions) {
        has_partitions = true;

        Partition_info partition;
        partition.info = &p;
        partition.basename = get_basename(
            common::encode_partition_basename(schema.name, table.name, p.name));

        table.partitions.emplace_back(std::move(partition));
      }

      schema.tables.emplace_back(std::move(table));
    }

    for (const auto &v : s.second.views) {
      View_info view;
      view.name = v.first;
      view.quoted_name = quote(schema.name, view.name);
      view.basename =
          get_basename(common::encode_table_basename(schema.name, view.name));
      view.info = &v.second;

      schema.views.emplace_back(std::move(view));
    }

    m_schema_infos.emplace_back(std::move(schema));
  }

  if (has_partitions) {
    m_used_capabilities.emplace(Capability::PARTITION_AWARENESS);
  }

  if (m_cache.has_library_ddl) {
    m_used_capabilities.emplace(Capability::MULTIFILE_SCHEMA_DDL);
  }

  if (m_options.handle_innodb_vector_store_tables()) {
    if (m_cache.has_innodb_vector_store_tables) {
      // WL16802-FR1.1.6: use the vector store-related capability
      m_used_capabilities.emplace(Capability::INNODB_VECTOR_STORE);

      // WL16802-FR1.1.1: if dump has InnoDB-based vector store tables, but user
      // did not provide lakehouseTarget option, we dump into two
      // subdirectories, one holding just the data of vector store tables, and
      // this one, holding the full dump
      if (!m_options.has_lakehouse_target()) {
        m_redirected_dump_dir = "dump";
      }
    } else {
      // WL16802-FR1.2.4: if there are no InnoDB-based vector store tables to be
      // dumped, but `lakehouseTarget` was given, print a note
      if (m_options.has_lakehouse_target()) {
        current_console()->print_note(
            "No InnoDB-based vector store tables included in the dump, "
            "'lakehouseTarget' option ignored.");
      }
    }
  } else {
    // WL16802-FR3: if vector store handling is disabled but MDS compatibility
    // checks are enabled, then this is a copy operation, print a warning that
    // auto conversion is not supported
    if (m_options.mds_compatibility() &&
        m_cache.has_innodb_vector_store_tables) {
      current_console()->print_warning(
          "Copy utilities currently do not support automatic conversion of "
          "InnoDB-based vector store tables to Lakehouse.");
    }
  }

  if (should_dump_data_masking_policies() || will_dump_masked_table_ddl()) {
    // WL17279-FR1.4: use capability if dump contains dynamic data masking DDL:
    // either policies or table DDL with masked columns
    m_used_capabilities.emplace(Capability::DYNAMIC_DATA_MASKING);
  }

  for (const auto capability : m_used_capabilities) {
    m_capability_set.set(capability);
  }
}

void Dumper::validate_mds() {
  if (!m_options.mds_compatibility() ||
      (!m_options.dump_ddl() && !dump_users())) {
    return;
  }

  const auto stage_attrib = [](const char *status) {
    auto ocimds_progress = shcore::make_dict("status", status);
    return shcore::make_dict("ocimdsProgress", std::move(ocimds_progress));
  };

  const auto console = current_console();
  const auto version = m_options.target_version().get_base();
  issues::Status_set status;

  console->print_note(
      "When migrating to MySQL HeatWave Service, please always use the latest "
      "available version of MySQL Shell.");

  console->print_info(
      "Checking for compatibility with MySQL HeatWave Service " + version,
      stage_attrib("begin"));

  if (!m_cache.server.version.is_8_0) {
    console->print_note("MySQL Server " +
                        m_cache.server.version.number.get_short() +
                        " detected, please consider upgrading to 8.0 first.");
  }

  console->print_info("Checking for potential upgrade issues.");
  status.set(check_for_upgrade_errors());

  {
    // setup progress reporting
    Progress_thread::Progress_config config;
    std::atomic<uint64_t> objects_checked{0};

    config.current = [&objects_checked]() -> uint64_t {
      return objects_checked;
    };

    {
      uint64_t objects_total = 0;

      if (dump_users()) {
        ++objects_total;
      }

      if (m_options.dump_ddl()) {
        objects_total += m_total_schemas;
        // tables + triggers
        objects_total += 2 * m_total_tables;
        // temporary + final views
        objects_total += 2 * m_total_views;
      }

      config.total = [objects_total]() { return objects_total; };
    }

    const auto stage = m_progress_thread.start_stage(
        "Validating MySQL HeatWave Service compatibility", std::move(config));
    shcore::on_leave_scope finish_stage([stage]() { stage->finish(); });

    // run compatibility checks using a thread pool
    shcore::Thread_pool_base<std::shared_ptr<Memory_dumper>> pool{
        m_options.threads()};

    const auto producer = [this](auto &&produce) {
      return [this, produce = std::move(produce)]() {
        Scoped_session session{session_pool()};
        const auto dumper = schema_dumper(session.get());

        return produce(dumper.get());
      };
    };

    const auto consumer = [this, &status, &objects_checked]() {
      return [this, &status, &objects_checked](auto &&memory) {
        status.set(show_issues(memory->issues()));
        ++objects_checked;
      };
    };

    pool.start_threads();

    if (dump_users()) {
      pool.add_task(
          producer([this](auto dumper) { return dump_users(dumper); }),
          consumer());
    }

    if (m_options.dump_ddl()) {
      for (const auto &schema : m_schema_infos) {
        // we're dumping whole schema DDL to a single file, it doesn't matter if
        // dump is using a single-file or multi-file schema, the contents is the
        // same
        pool.add_task(producer([this, &schema = schema.name](auto dumper) {
                        return dump_complete_schema(dumper, schema);
                      }),
                      consumer());
      }

      const auto dumping_triggers = [this](const auto &schema,
                                           const auto &table) {
        return m_options.dump_triggers() &&
               m_cache.schemas.at(schema).tables.at(table).triggers.size();
      };

      for (const auto &schema : m_schema_infos) {
        for (const auto &table : schema.tables) {
          pool.add_task(producer([this, &schema = schema.name,
                                  &table = table.name](auto dumper) {
                          return dump_table(dumper, schema, table);
                        }),
                        consumer());

          if (dumping_triggers(schema.name, table.name)) {
            pool.add_task(producer([this, &schema = schema.name,
                                    &table = table.name](auto dumper) {
                            return dump_triggers(dumper, schema, table);
                          }),
                          consumer());
          } else {
            // increase the count, marking this task as complete
            ++objects_checked;
          }
        }

        for (const auto &view : schema.views) {
          pool.add_task(producer([this, &schema = schema.name,
                                  &view = view.name](auto dumper) {
                          return dump_temporary_view(dumper, schema, view);
                        }),
                        consumer());

          pool.add_task(producer([this, &schema = schema.name,
                                  &view = view.name](auto dumper) {
                          return dump_view(dumper, schema, view);
                        }),
                        consumer());
        }
      }
    }

    pool.tasks_done();
    pool.process();
  }

  if (m_options.implicit_target_version() &&
      status.is_set(issues::Status::WARNING_DEPRECATED_DEFINERS)) {
    console->print_info();
    console->print_note(
        "One or more objects with the DEFINER clause were found.");
    console->print_info(
        shcore::str_format(R"(
      The 'targetVersion' option was not set and compatibility was checked with the MySQL HeatWave Service %s.
      Loading the dump will fail if it is loaded into an DB System instance that does not support the SET_ANY_DEFINER privilege, which was introduced in 8.2.0.
)",
                           m_options.target_version().get_base().c_str()));
  }

  if (status.is_set(issues::Status::ERROR_MISSING_PKS)) {
    console->print_info();
    console->print_error("One or more tables without Primary Keys were found.");
    console->print_info(shcore::str_format(
        R"(
       MySQL HeatWave Service High Availability (MySQL HeatWave Service HA) requires Primary Keys%s to be present in all tables.
       To continue with the dump you must do one of the following:

       * Create PRIMARY keys (regular or invisible) in all tables before dumping them.
         MySQL 8.0.23 supports the creation of invisible columns to allow creating Primary Key columns with no impact to applications. For more details, see https://dev.mysql.com/doc/refman/en/invisible-columns.html.
         This is considered a best practice for both performance and usability and will work seamlessly with MySQL HeatWave Service.

       * Add the "create_invisible_pks" to the "compatibility" option.
         The dump will proceed and loader will automatically add Primary Keys to tables that don't have them when loading into MySQL HeatWave Service.
         This will make it possible to enable HA in MySQL HeatWave Service without application impact and without changes to the source database.
         Inbound Replication into a DB System HA instance will also be possible, as long as the instance has version 8.0.32 or newer. For more information, see https://docs.oracle.com/en-us/iaas/mysql-database/doc/creating-replication-channel.html.

       * Add the "ignore_missing_pks" to the "compatibility" option.
         This will disable this check and the dump will be produced normally, Primary Keys will not be added automatically.
         It will not be possible to load the dump in an HA enabled DB System instance.
)",
        compatibility::supports_pke_as_pk(m_options.target_version())
            ? " or Primary Key Equivalents"
            : ""));
  }

  if (status.is_set(issues::Status::ERROR_HAS_WILDCARD_GRANTS)) {
    console->print_info();
    console->print_error(
        "One or more accounts with database level grants containing wildcard "
        "characters were found.");
    console->print_info(shcore::str_format(
        R"(
      Loading these grants into a DB System instance may lead to unexpected results, as the partial_revokes system variable is enabled, which in turn changes the interpretation of wildcards in GRANT statements.
      To continue with the dump you must do one or more of the following:

      * Manually update the schema names in GRANT statements of problematic accounts.

      * Use the "excludeUsers" dump option to exclude problematic accounts.%s

      * Add "ignore_wildcard_grants" to the "compatibility" option to ignore these issues.

      * Set the "users" dump option to false in order to disable user dumping.

      For more information on the interaction between wildcard database level grants and partial_revokes system variable please refer to: https://dev.mysql.com/doc/refman/en/grant.html
    )",
        status.is_set(issues::Status::WARNING_ESCAPED_WILDCARDS) ? R"(

      * Add "unescape_wildcard_grants" to the "compatibility" option to automatically replace escaped \_ and \% wildcards in schema names with _ and % wildcard characters.)"
                                                                 : ""));
  }

  if (status.is_set(issues::Status::ERROR_HAS_NON_STANDARD_FKS)) {
    console->print_info();
    console->print_error(
        "One or more tables with non-standard foreign keys (that reference "
        "non-unique keys or partial fields of composite keys) were found.");
    console->print_info(R"(
      Loading these tables into a DB System instance may fail, as instances are created with the 'restrict_fk_on_non_standard_key' system variable set to 'ON'.
      To continue with the dump you must do one of the following:

      * Fix the tables listed above, i.e. create the missing unique key.

      * Add the "force_non_standard_fks" to the "compatibility" option to ignore these issues. Please note that creation of foreign keys with non-standard keys may break the replication.
    )");
  }

  // this is an error in case of MHS
  handle_mismatched_view_references(status, true);

  if (status.is_set(issues::Status::ERROR)) {
    console->print_info(
        "Compatibility issues with MySQL HeatWave Service " + version +
            " were found. Please use the 'compatibility' option to apply "
            "compatibility adaptations to the dumped DDL.",
        stage_attrib("error"));
    THROW_ERROR(SHERR_DUMP_COMPATIBILITY_ISSUES_FOUND);
  }

  if (status.matches_any(
          issues::Status_set{issues::Status::FIXED_CREATE_PKS,
                             issues::Status::FIXED_IGNORE_PKS})) {
    console->print_info();
    console->print_note("One or more tables without Primary Keys were found.");
  }

  if (status.is_set(issues::Status::FIXED_CREATE_PKS)) {
    console->print_info(R"(
      Missing Primary Keys will be created automatically when this dump is loaded.
      This will make it possible to enable High Availability in MySQL HeatWave Service DB System instance without application impact and without changes to the source database.
      Inbound Replication into a DB System HA instance will also be possible, as long as the instance has version 8.0.32 or newer. For more information, see https://docs.oracle.com/en-us/iaas/mysql-database/doc/creating-replication-channel.html.
)");
  }

  if (status.is_set(issues::Status::FIXED_IGNORE_PKS)) {
    console->print_info(R"(
      This issue is ignored.
      This dump cannot be loaded into an MySQL HeatWave Service DB System instance with High Availability.
)");
  }

  if (status.is_set(issues::Status::FIXED)) {
    console->print_info(
        "Compatibility issues with MySQL HeatWave Service " + version +
            " were found and repaired. Please review the changes "
            "made before loading them.",
        stage_attrib("fixed"));
  }

  if (status.empty()) {
    console->print_info("Compatibility checks finished.", stage_attrib("end"));
  }
}

void Dumper::validate_data_masking() {
  if (!will_dump_table_data_with_masked_columns()) {
    return;
  }

  if (m_user_privileges->validate({"MANAGE_DATA_MASKING_POLICY"})
          .has_missing_privileges()) {
    if (m_options.allow_data_masking()) {
      // WL17279-FR1.2.4: 'allowDataMasking' is enabled, but user doesn't
      // have the required privilege, warn and skip the check
      current_console()->print_warning(
          "Dump contains tables with masked columns, current user does not "
          "have the 'MANAGE_DATA_MASKING_POLICY' privilege and the "
          "'allowDataMasking' option is enabled, skipping unmasked data access "
          "check.");
      return;
    } else {
      // WL17279-FR1.3: MANAGE_DATA_MASKING_POLICY privilege is needed when we
      // need to ensure that dumped data is not masked
      THROW_ERROR(SHERR_DUMP_MISSING_MANAGE_DATA_MASKING_POLICY_PRIVILEGE);
    }
  }

  report_data_masking_issues(
      verify_data_masking_policies(list_data_masking_policies_in_use()));
}

std::unordered_set<std::string> Dumper::list_data_masking_policies_in_use() {
  shcore::Thread_pool_base<std::unordered_set<std::string>> pool{
      m_options.threads()};
  std::unordered_set<std::string> result;

  const auto producer = [this](const std::string &table) {
    return [this, &table]() {
      Scoped_session session{session_pool()};
      const auto create_table = session.get()
                                    ->query("SHOW CREATE TABLE " + table)
                                    ->fetch_one_or_throw()
                                    ->get_string(1);

      return compatibility::list_data_masking_policies(create_table);
    };
  };

  const auto consumer = [&result]() {
    return [&result](std::unordered_set<std::string> &&policies) {
      result.merge(std::move(policies));
    };
  };

  pool.start_threads();

  for (const auto &schema : m_schema_infos) {
    for (const auto &table : schema.tables) {
      if (table.info->has_masking_policy) {
        pool.add_task(producer(table.quoted_name), consumer());
      }
    }
  }

  pool.tasks_done();
  pool.process();

  return result;
}

namespace {

template <typename Key, typename Value>
class Synchronized_cache final {
 public:
  Synchronized_cache() = default;

  Synchronized_cache(const Synchronized_cache &) = delete;
  Synchronized_cache(Synchronized_cache &&) = delete;

  Synchronized_cache &operator=(const Synchronized_cache &) = delete;
  Synchronized_cache &operator=(Synchronized_cache &&) = delete;

  ~Synchronized_cache() = default;

  std::optional<Value> get(const Key &k) const {
    std::shared_lock lock{m_mutex};

    if (const auto it = m_cache.find(k); it != m_cache.end()) {
      return it->second;
    } else {
      return std::nullopt;
    }
  }

  void set(const Key &k, const Value &v) {
    std::unique_lock lock{m_mutex};

    m_cache.insert_or_assign(k, v);
  }

  void set(const Key &k, Value &&v) {
    std::unique_lock lock{m_mutex};

    m_cache.insert_or_assign(k, std::move(v));
  }

 private:
  mutable std::shared_mutex m_mutex;
  std::unordered_map<Key, Value> m_cache;
};

}  // namespace

std::vector<mysqlsh::common::Data_masking::Policy>
Dumper::verify_data_masking_policies(std::unordered_set<std::string> &&names) {
  if (names.empty()) {
    return {};
  }

  using Data_masking = mysqlsh::common::Data_masking;
  using Policy = Data_masking::Policy;

  shcore::Thread_pool_base<Policy> pool{m_options.threads()};
  std::vector<Policy> result;

  Synchronized_cache<std::string_view, bool> cache;

  const auto producer = [this, &cache](std::string &&policy) {
    return [this, &cache, policy = std::move(policy)]() -> Policy {
      Scoped_session session{session_pool()};

      auto p = Data_masking{session.get()}.parse_policy(policy);
      auto allow = cache.get(p.gatekeeper);

      if (!allow.has_value()) {
        allow = session.get()
                    ->query("SELECT " + p.gatekeeper)
                    ->fetch_one()
                    ->get_int(0);

        cache.set(p.gatekeeper, allow.value());
      }

      if (allow.value() == p.allow) {
        return {};
      } else {
        // account used for the dump does not have access to the unmasked data
        return p;
      }
    };
  };

  const auto consumer = [&result]() {
    return [&result](Policy &&policy) {
      // if name is non-empty then policy is not satisfied
      if (!policy.name.empty()) {
        result.emplace_back(std::move(policy));
      }
    };
  };

  pool.start_threads();

  while (!names.empty()) {
    pool.add_task(producer(std::move(names.extract(names.begin()).value())),
                  consumer());
  }

  pool.tasks_done();
  pool.process();

  return result;
}

namespace {

using Policy = mysqlsh::common::Data_masking::Policy;

/**
 * Split string at unquoted commas, remove any unquoted spaces. Parse each
 * segment as an account, ignore any errors.
 */
std::set<std::string> split_user_list(std::string_view user_list_in) {
  // an empty string is at least two characters: opening and closing quote
  if (user_list_in.length() <= 2) {
    return {};
  }

  const auto user_list =
      shcore::unquote_sql_string(user_list_in, user_list_in.front());

  std::set<std::string> result;
  std::string account;
  account.reserve(user_list.length());

  const auto add_account = [&result, &account]() {
    if (account.empty()) {
      return;
    }

    try {
      auto a = shcore::split_account(account);

      // if host is not given we assume it's '%', this is also true in case of
      // roles
      if (a.host.empty()) {
        a.host = "%";
      }

      account = shcore::make_account(a);
    } catch (...) {
      // user list is not verified when creating the policy, ignore any errors
      // and use the string provided by the user
    }

    result.emplace(std::move(account));

    account.clear();
  };

  std::size_t i = 0;
  const auto end = user_list.length();

  const auto copy_quoted = [&user_list, &i, &account, end](const auto &f) {
    const auto e = f(user_list, i);

    if (std::string_view::npos == e) {
      account += user_list.substr(i);
      i = end;
    } else {
      account += user_list.substr(i, e - i);
      i = e;
    }
  };

  char c;

  while (i < end) {
    c = user_list[i];

    switch (c) {
      case '\'':
        copy_quoted(mysqlshdk::utils::span_quoted_string_sq);
        break;

      case '"':
        copy_quoted(mysqlshdk::utils::span_quoted_string_dq);
        break;

      case '`':
        copy_quoted(mysqlshdk::utils::span_quoted_sql_identifier_bt);
        break;

      case ',':
        add_account();
        ++i;
        break;

      case ' ':
        // skip spaces
        ++i;
        break;

      default:
        // append to account
        account += c;
        ++i;
        break;
    }
  }

  add_account();

  return result;
}

std::set<std::string> extract_user_list(std::string_view gatekeeper) {
  static constexpr std::string_view k_quotes = "'\"";

  const auto start = gatekeeper.find_first_of(k_quotes);
  const auto end = gatekeeper.find_last_of(k_quotes);

  return split_user_list(gatekeeper.substr(start, end - start + 1));
}

std::string format_policy(const Policy &policy) {
  using Authorization_type = Policy::Authorization_type;

  std::string str;

  str.reserve(64);

  str += "Policy ";
  str += shcore::quote_identifier(policy.name);
  str += ' ';
  str += policy.allow ? "allows" : "denies";
  str += " access for ";

  switch (policy.authorization_type) {
    case Authorization_type::USER:
      str += "user";
      break;

    case Authorization_type::ROLE:
      str += "role";
      break;

    case Authorization_type::UNKNOWN:
      assert(false);
      break;
  }

  const auto ids = extract_user_list(policy.gatekeeper);

  if (ids.size() > 1) {
    str += 's';
  }

  str += ": ";
  str += shcore::str_join(ids, ", ");

  return str;
}

}  // namespace

void Dumper::report_data_masking_issues(
    const std::vector<Policy> &policies) const {
  if (policies.empty()) {
    return;
  }

  // use map to sort the policies by name
  std::map<std::string_view, std::reference_wrapper<const Policy>> errors;

  for (const auto &policy : policies) {
    errors.emplace(policy.name, std::ref(policy));
  }

  const auto console = current_console();
  const auto &active_roles = m_user_privileges->get_user_roles();
  const auto as_error = !m_options.allow_data_masking();

  // WL17279-FR1.2.3: 'allowDataMasking' is enabled, warn instead of reporting
  // an error
  std::invoke(as_error ? &IConsole::print_error : &IConsole::print_warning,
              console,
              "One or more data masking policies deny access to table data.",
              IConsole::Json_attributes{});

  console->print_info("");
  console->print_info(shcore::str_format(
      R"(Dump contains tables which use dynamic data masking, while current user %s %s not have access to unmasked data due to the following policies:
 * %s
)",
      shcore::make_account(m_user_account).c_str(),
      active_roles.empty()
          ? "does"
          : shcore::str_format("and its active role%s (%s) do",
                               active_roles.size() > 1 ? "s" : "",
                               shcore::str_join(active_roles, ", ").c_str())
                .c_str(),
      shcore::str_join(errors, "\n * ", [](const auto &p) {
        return format_policy(p.second);
      }).c_str()));

  if (as_error) {
    console->print_info(
        R"(To continue with the dump you must do one of the following:
 * Use an account which fulfills all the listed policies and ensure that all required roles are activated when a new session is created.
 * Enable the "allowDataMasking" dump option to ignore this issue. Please note that dump will contain masked data.
)");

    // WL17279-FR1.2.2: 'allowDataMasking' is disabled, report an error
    THROW_ERROR(SHERR_DUMP_CONTAINS_MASKED_TABLE_DATA);
  } else {
    console->print_info(
        R"(The "allowDataMasking" dump option is enabled, ignoring this issue.
)");
  }
}

void Dumper::initialize_counters() {
  m_total_rows = 0;
  m_total_data_size = 0;
  m_total_tables = 0;
  m_total_views = 0;
  m_total_schemas = m_schema_infos.size();

  for (auto &schema : m_schema_infos) {
    m_total_tables += schema.tables.size();
    m_total_views += schema.views.size();

    for (auto &table : schema.tables) {
      m_total_rows += table.info->row_count;
      m_total_data_size +=
          table.info->row_count * table.info->average_row_length;
    }
  }

  m_total_objects = m_total_schemas + m_total_tables + m_total_views;

  m_rows_written = 0;
  m_bytes_written = 0;
  m_data_bytes = 0;
  m_table_data_stats.clear();

  m_data_throughput = std::make_unique<mysqlshdk::textui::Throughput>();
  m_bytes_throughput = std::make_unique<mysqlshdk::textui::Throughput>();

  m_num_threads_chunking = 0;
  m_num_threads_checksumming = 0;
  m_num_threads_dumping = 0;

  m_ddl_to_write = m_options.dump_ddl() ? m_total_objects : 0;
  m_ddl_written = 0;

  const auto write_metadata = should_write_metadata();

  m_schema_metadata_to_write = write_metadata ? m_total_schemas : 0;
  m_schema_metadata_written = 0;

  m_table_metadata_to_write = write_metadata ? m_total_tables : 0;
  m_table_metadata_written = 0;
}

void Dumper::initialize_dump() {
  if (Dry_run::DONT_WRITE_ANY_FILES == m_options.dry_run_mode()) {
    return;
  }

  create_output_directory();
  write_dump_started_metadata();
}

void Dumper::finalize_dump() {
  if (Dry_run::DONT_WRITE_ANY_FILES == m_options.dry_run_mode()) {
    return;
  }

  write_checksum_metadata();
  write_dump_finished_metadata();
  close_output_directory();
}

void Dumper::create_output_directory() {
  if (!dump_dir()) {
    return;
  }

  m_dump_config.create_directory();

  if (uses_dump_dir_redirection()) {
    write_dump_redirected_metadata();

    // change output directory
    m_redirected_output_url =
        dump_dir()->join_path(m_options.output_url(), m_redirected_dump_dir);
    ensure_prefix_par(&m_redirected_output_url);

    m_dump_config.dir = dump_dir()->directory(m_redirected_dump_dir);

    // make sure that the redirected output directory exists
    m_dump_config.create_directory();
  }

  setup_vector_store_directory();
}

void Dumper::close_output_directory() {
  if (Dry_run::DONT_WRITE_ANY_FILES == m_options.dry_run_mode()) {
    return;
  }

  Progress_thread::Stage *stage = nullptr;
  shcore::on_leave_scope finish_stage([&stage]() {
    if (stage) {
      stage->finish();
    }
  });

  close_vector_store_directory();
  m_dump_config.fini();
}

void Dumper::setup_vector_store_directory() {
  if (!uses_innodb_vector_store()) {
    return;
  }

  using Storage_type = mysqlsh::common::Storage_options::Storage_type;

  bool relative_path = true;
  std::string output_url;
  Storage_type storage_type = Storage_type::Unknown;
  mysqlshdk::storage::Config_ptr storage_config;

  if (m_options.has_lakehouse_target()) {
    const auto &target = m_options.lakehouse_target();

    if (Storage_type::Local == target.storage_type() &&
        mysqlshdk::storage::utils::get_scheme(target.output_url()).empty()) {
      // WL16802-FR1.2.1.3: if storage type is local, then the value of
      // `lakehouseTarget.outputUrl` doesn't have a scheme or has a 'file://'
      // scheme and `lakehouseTarget` doesn't have any of the remote options; in
      // such case we use the same storage as the whole dump

      // identify the storage used by the dump
      storage_config =
          m_options.storage_config(m_options.output_url(), &storage_type);

      if (Storage_type::Oci_prefix_par == storage_type) {
        // this is a prefix PAR, we need to construct PAR with the new prefix
        const auto par_config =
            std::dynamic_pointer_cast<const mysqlshdk::oci::IPAR_config>(
                storage_config);

        assert(par_config);

        output_url = par_config->par().par_url() + target.output_url();
      } else {
        // in any other case, we get the URL from `lakehouseTarget` and use
        // dump's storage config
        output_url = target.output_url();
      }
    } else {
      // WL16802-FR1.2.5: use storage provided via `lakehouseTarget`
      output_url = target.output_url();
      m_vector_store_config.dir = target.output();
      m_vector_store_output_description = target.describe_output_url();
      storage_type = target.storage_type();
      storage_config = target.storage_config();
      relative_path = false;
    }
  } else {
    // WL16802-FR1.1: user did not provide the storage option, dump to a
    // subdirectory
    output_url = dump_dir()->join_path(m_options.output_url(), "vector-store");
  }

  if (!vector_store_dir()) {
    ensure_prefix_par(&output_url);

    m_vector_store_config.dir = m_options.make_directory(output_url);
    m_vector_store_output_description = m_options.describe(output_url);
    storage_config = m_options.storage_config(output_url, &storage_type);
  }

  if (relative_path) {
    // if relative path is available, loader will be able load the InnoDB-based
    // vector store tables with no conversion
    m_vector_store_info.relative_path = shcore::path::get_relative_path(
        dump_dir()->full_path().real(), vector_store_dir()->full_path().real());
  }

  // if vector data is stored in a PAR or a bucket, loader will be able to
  // convert InnoDB-based vector store tables to Lakehouse even if
  // 'lakehouseSource' is not given
  if (Storage_type::Oci_os == storage_type) {
    const auto oci_config =
        std::dynamic_pointer_cast<const mysqlshdk::oci::Oci_bucket_config>(
            storage_config);

    assert(oci_config);

    m_vector_store_info.resource_principals.bucket = oci_config->bucket_name();
    m_vector_store_info.resource_principals.namespace_ =
        oci_config->oci_namespace();
    m_vector_store_info.resource_principals.region =
        oci_config->credentials_provider()->region();
    m_vector_store_info.resource_principals.set_prefix(std::move(output_url));
  } else if (Storage_type::Oci_prefix_par == storage_type) {
    const auto par_config =
        std::dynamic_pointer_cast<const mysqlshdk::oci::IPAR_config>(
            storage_config);

    assert(par_config);

    const auto &par = par_config->par();

    m_vector_store_info.resource_principals.bucket = par.bucket();
    m_vector_store_info.resource_principals.namespace_ = par.namespace_();
    m_vector_store_info.resource_principals.region = par.region();
    m_vector_store_info.resource_principals.set_prefix(par.object_prefix() +
                                                       par.object_name());
  }

  // WL16802-FR1.1.3: csv-unix dialect
  // WL16802-FR1.1.4: gzip compression, if dump is compressed
  m_vector_store_config.init(import_table::Dialect::csv_unix(),
                             compressed()
                                 ? mysqlshdk::storage::Compression::GZIP
                                 : mysqlshdk::storage::Compression::NONE,
                             {});

  // create vector store directory
  m_vector_store_config.create_directory();

  if (!m_vector_store_config.dir->is_empty()) {
    // BUG#38219104 - error out if vector store directory is not empty
    const auto masked_url = mysqlshdk::oci::mask_any_par(output_url);

    if (storage_config && storage_config->valid()) {
      throw std::invalid_argument(
          "Invalid value of 'lakehouseTarget' option, " +
          storage_config->description() +
          " already contains files with the specified prefix '" + masked_url +
          "'.");
    } else {
      throw std::invalid_argument(
          "Invalid value of 'lakehouseTarget' option, the specified directory "
          "'" +
          masked_url + "' already exists at the target location " +
          m_vector_store_config.dir->full_path().masked() +
          " and is not empty.");
    }
  }
}

void Dumper::close_vector_store_directory() { m_vector_store_config.fini(); }

void Dumper::create_worker_sessions() {
  for (std::size_t i = 0; i < m_options.threads(); ++i) {
    auto worker_session =
        establish_session(session()->get_connection_options(), false);

    start_transaction(worker_session);
    on_init_thread_session(worker_session);

    m_session_pool.push(std::move(worker_session));
  }
}

void Dumper::create_worker_threads() {
  m_worker_exceptions.clear();
  m_worker_exceptions.resize(m_options.worker_threads());

  for (std::size_t i = 0; i < m_options.worker_threads(); ++i) {
    auto t = mysqlsh::spawn_scoped_thread(
        &Table_worker::run,
        Table_worker{i, this, Table_worker::Exception_strategy::ABORT});
    m_workers.emplace_back(std::move(t));
  }
}

void Dumper::maybe_push_shutdown_tasks() {
  if (all_tasks_produced()) {
    m_worker_tasks.shutdown(m_workers.size());
  }
}

void Dumper::chunking_task_finished() {
  ++m_chunking_tasks_completed;
  maybe_push_shutdown_tasks();
}

void Dumper::data_task_finished() {
  ++m_data_tasks_completed;

  if (m_data_dump_stage && all_tasks_produced() &&
      m_data_tasks_completed == m_data_tasks_total) {
    m_data_dump_stage->finish(false);
  }
}

void Dumper::checksum_task_started() {
  bool expected = false;

  if (m_checksum_started.compare_exchange_strong(expected, true)) {
    m_checksum_duration.start();
  }
}

void Dumper::checksum_task_finished() {
  ++m_checksum_tasks_completed;

  if (m_checksum_tasks_total == m_checksum_tasks_completed) {
    // we don't check here if all tasks were produced, there can be i.e. only
    // a single checksum task that finishes executing before all tasks are
    // produced, while calling finish() multiple times is harmless
    m_checksum_duration.finish();
  }
}

void Dumper::wait_for_all_tasks() {
  for (auto &worker : m_workers) {
    worker.join();
  }

  m_workers.clear();

  // when using a single file as an output, it's not closed until the whole
  // dump is done
  if (m_output_file && m_output_file->is_open()) {
    m_output_file->close();
  }
}

void Dumper::dump_ddl() const {
  if (!m_options.dump_ddl()) {
    return;
  }

  dump_global_ddl();
  dump_users_ddl();
  dump_data_masking_policies_ddl();
}

void Dumper::dump_global_ddl() const {
  current_console()->print_status("Writing global DDL files");

  if (Dry_run::DONT_WRITE_ANY_FILES == m_options.dry_run_mode()) {
    return;
  }

  const auto dumper = schema_dumper(session());

  {
    // file with the DDL setup
    const auto output = make_dump_file(common::k_preamble_sql_file);
    output->open(Mode::WRITE);

    dumper->write_comment(output.get());

    output->close();
  }

  {
    // post DDL file (cleanup)
    const auto output = make_dump_file(common::k_postamble_sql_file);
    output->open(Mode::WRITE);

    dumper->write_comment(output.get());

    output->close();
  }
}

void Dumper::dump_users_ddl() const {
  if (!dump_users()) {
    return;
  }

  current_console()->print_status("Writing users DDL");

  const auto dumper = schema_dumper(session());

  write_ddl(*dump_users(dumper.get()), common::k_users_sql_file);
}

void Dumper::dump_data_masking_policies_ddl() const {
  if (!should_dump_data_masking_policies()) {
    return;
  }

  current_console()->print_status("Writing data masking policies DDL");

  const auto dumper = schema_dumper(session());

  write_ddl(*dump_data_masking_policies(dumper.get()),
            common::k_data_masking_file);
}

void Dumper::write_ddl(const Memory_dumper &in_memory,
                       const std::string &file) const {
  if (!m_options.mds_compatibility()) {
    // if MDS is on, changes done by compatibility options were printed earlier
    const auto status = show_issues(in_memory.issues());

    handle_mismatched_view_references(status, false);

    {
      std::lock_guard lock{m_compatibility_status_mutex};
      m_compatibility_status.set(status);

      if (m_compatibility_status.is_set(issues::Status::ERROR)) {
        // fatal error, dump will be aborted
        return;
      }
    }
  }

  if (Dry_run::DONT_WRITE_ANY_FILES == m_options.dry_run_mode()) {
    return;
  }

  const auto output = make_dump_file(file);
  output->open(Mode::WRITE);

  const auto &content = in_memory.content();
  output->write(content.c_str(), content.length());

  output->close();
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_ddl(
    Schema_dumper *dumper,
    const std::function<void(Memory_dumper *)> &func) const {
  auto memory = std::make_unique<Memory_dumper>(dumper);

  memory->dump(func);

  return memory;
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_complete_schema(
    Schema_dumper *dumper, const std::string &schema) const {
  return dump_ddl(dumper, [&schema, this](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, std::string{});
    m->dump(&Schema_dumper::dump_schema_ddl, schema);

    if (m_options.dump_events()) {
      m->dump(&Schema_dumper::dump_events_ddl, schema);
    }

    if (m_options.dump_libraries()) {
      m->dump(&Schema_dumper::dump_libraries_ddl, schema);
    }

    if (m_options.dump_routines()) {
      m->dump(&Schema_dumper::dump_routines_ddl, schema);
    }
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_schema(
    Schema_dumper *dumper, const std::string &schema) const {
  return dump_ddl(dumper, [&schema](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, std::string{});
    m->dump(&Schema_dumper::dump_schema_ddl, schema);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_events(
    Schema_dumper *dumper, const std::string &schema) const {
  return dump_ddl(dumper, [&schema](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, std::string{});
    m->dump(&Schema_dumper::dump_events_ddl, schema);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_libraries(
    Schema_dumper *dumper, const std::string &schema) const {
  return dump_ddl(dumper, [&schema](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, std::string{});
    m->dump(&Schema_dumper::dump_libraries_ddl, schema);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_routines(
    Schema_dumper *dumper, const std::string &schema) const {
  return dump_ddl(dumper, [&schema](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, std::string{});
    m->dump(&Schema_dumper::dump_routines_ddl, schema);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_table(
    Schema_dumper *dumper, const std::string &schema,
    const std::string &table) const {
  return dump_ddl(dumper, [&schema, &table](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, table);
    m->dump(&Schema_dumper::dump_table_ddl, schema, table);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_triggers(
    Schema_dumper *dumper, const std::string &schema,
    const std::string &table) const {
  return dump_ddl(dumper, [&schema, &table](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, table);
    m->dump(&Schema_dumper::dump_triggers_for_table_ddl, schema, table);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_temporary_view(
    Schema_dumper *dumper, const std::string &schema,
    const std::string &view) const {
  return dump_ddl(dumper, [&schema, &view](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, view);
    m->dump(&Schema_dumper::dump_temporary_view_ddl, schema, view);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_view(
    Schema_dumper *dumper, const std::string &schema,
    const std::string &view) const {
  return dump_ddl(dumper, [&schema, &view](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, schema, view);
    m->dump(&Schema_dumper::dump_view_ddl, schema, view);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_users(
    Schema_dumper *dumper) const {
  return dump_ddl(dumper, [](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, std::string{}, std::string{});
    m->dump(&Schema_dumper::dump_grants);
  });
}

std::unique_ptr<Dumper::Memory_dumper> Dumper::dump_data_masking_policies(
    Schema_dumper *dumper) const {
  return dump_ddl(dumper, [](Memory_dumper *m) {
    m->dump(&Schema_dumper::write_comment, std::string{}, std::string{});
    m->dump(&Schema_dumper::dump_data_masking_policies);
  });
}

void Dumper::create_schema_metadata_tasks() {
  if (!should_write_metadata()) {
    return;
  }

  Progress_thread::Progress_config config;
  config.current = [this]() -> uint64_t { return m_schema_metadata_written; };
  config.total = [this]() { return m_schema_metadata_to_write; };

  m_progress_thread.start_stage("Writing schema metadata", std::move(config));

  for (const auto &schema : m_schema_infos) {
    m_worker_tasks.push({"writing metadata of " + schema.quoted_name,
                         [&schema](Table_worker *worker) {
                           worker->write_schema_metadata(schema);
                         }},
                        shcore::Queue_priority::HIGH);
  }
}

void Dumper::create_schema_ddl_tasks() {
  if (!m_options.dump_ddl()) {
    return;
  }

  Progress_thread::Progress_config config;

  config.current = [this]() -> uint64_t { return m_ddl_written; };
  config.total = [this]() { return m_ddl_to_write; };

  m_progress_thread.start_stage("Writing DDL", std::move(config));

  for (const auto &schema : m_schema_infos) {
    m_worker_tasks.push(
        {"writing DDL of " + schema.quoted_name,
         [&schema](Table_worker *worker) { worker->dump_schema_ddl(schema); }},
        shcore::Queue_priority::HIGH);

    for (const auto &view : schema.views) {
      m_worker_tasks.push({"writing DDL of " + view.quoted_name,
                           [&schema, &view](Table_worker *worker) {
                             worker->dump_view_ddl(schema, view);
                           }},
                          shcore::Queue_priority::HIGH);
    }

    for (const auto &table : schema.tables) {
      m_worker_tasks.push({"writing DDL of " + table.quoted_name,
                           [&schema, &table](Table_worker *worker) {
                             worker->dump_table_ddl(schema, table);
                           }},
                          shcore::Queue_priority::HIGH);
    }
  }
}

void Dumper::create_table_tasks() {
  m_chunking_tasks_total = 0;
  m_chunking_tasks_completed = 0;
  m_data_tasks_total = 0;
  m_data_tasks_completed = 0;
  m_checksum_started = false;
  m_checksum_tasks_total = 0;
  m_checksum_tasks_completed = 0;

  m_main_thread_finished_producing_chunking_tasks = false;

  const auto write_metadata = should_write_metadata();

  if (write_metadata) {
    Progress_thread::Progress_config config;

    config.current = [this]() -> uint64_t { return m_table_metadata_written; };
    config.total = [this]() -> uint64_t { return m_table_metadata_to_write; };

    m_progress_thread.start_stage("Writing table metadata", std::move(config));
  }

  const auto console = current_console();

  std::list<Table_task> pending_tasks;

  for (const auto &schema : m_schema_infos) {
    for (const auto &table : schema.tables) {
      auto task = create_table_task(schema, table);

      if (write_metadata) {
        m_worker_tasks.push({"writing metadata of " + table.quoted_name,
                             [task](Table_worker *worker) {
                               worker->write_table_metadata(task);
                             }},
                            shcore::Queue_priority::HIGH);
      }

      if (m_checksum) {
        m_checksum->initialize_table(task.schema, task.name, task.info,
                                     task.index.info, task.extra_filter);
      }

      pending_tasks.emplace_back(std::move(task));
    }

    // BUG#34663934 - allow exporting data from views
    if (m_options.is_export_only()) {
      for (const auto &view : schema.views) {
        pending_tasks.emplace_back(create_table_task(schema, view));
      }
    }
  }

  for (auto &task : pending_tasks) {
    push_table_task(std::move(task));
  }

  m_main_thread_finished_producing_chunking_tasks = true;
}

Dumper::Table_task Dumper::create_table_task(const Schema_info &schema,
                                             const Table_info &table) {
  Table_task task;
  task.task_name = table.quoted_name;
  task.name = table.name;
  task.quoted_name = table.quoted_name;
  task.schema = schema.name;
  task.basename = table.basename;
  task.info = table.info;
  std::tie(task.index.info, task.index.is_pke) = select_index(*table.info);
  task.partitions = table.partitions;
  task.extra_filter = m_options.where(schema.name, table.name);

  if (uses_innodb_vector_store() && table.info->is_innodb_vector_store_table) {
    task.output_config = &m_vector_store_config;
  } else {
    task.output_config = &m_dump_config;
  }

  on_create_table_task(task.schema, task.name, task.info);

  return task;
}

Dumper::Table_task Dumper::create_table_task(const Schema_info &schema,
                                             const View_info &view) {
  Table_task task;
  task.task_name = view.quoted_name;
  task.name = view.name;
  task.quoted_name = view.quoted_name;
  task.schema = schema.name;
  task.basename = view.basename;
  task.info = view.info;
  task.extra_filter = m_options.where(schema.name, view.name);
  task.output_config = &m_dump_config;

  on_create_table_task(task.schema, task.name, task.info);

  return task;
}

void Dumper::push_table_task(Table_task &&task) {
  bool dump = true;

  if (!m_options.dump_data()) {
    if (!m_checksum) {
      return;
    } else {
      // we're going to create checksum tasks
      dump = false;
    }
  }

  const auto context =
      shcore::str_format("%s of table %s", dump ? "data dump" : "checksum",
                         task.quoted_name.c_str());

  if (!should_dump_data(task)) {
    current_console()->print_warning("Skipping " + context);
    return;
  }

  log_info("Preparing %s", context.c_str());

  const auto index = task.index.info;

  const auto get_index = [&index]() {
    assert(index);
    return std::string("column") + (index->columns().size() > 1 ? "s" : "") +
           " " + index->columns_sql();
  };

  if (m_options.split()) {
    if (!index) {
      log_info(
          "Could not select columns to be used as an index for table %s. Data "
          "will be dumped to multiple files by a single thread.",
          task.quoted_name.c_str());
    } else {
      log_info("The %s will be %s using %s", context.c_str(),
               dump ? "chunked" : "ordered", get_index().c_str());
    }
  } else {
    const auto index_info =
        (!index ? "will not use an index"
                : "will use " + get_index() + " as an index");
    log_info("The %s %s", context.c_str(), index_info.c_str());
  }

  if (Dry_run::DONT_WRITE_ANY_FILES == m_options.dry_run_mode()) {
    return;
  }

  std::vector<Table_task> tasks;
  tasks.reserve(task.partitions.size());

  if (m_options.is_export_only() || task.partitions.empty()) {
    tasks.emplace_back(std::move(task));
  } else {
    for (const auto &partition : task.partitions) {
      auto copy = task;

      copy.task_name =
          task.task_name + " partition " + partition.info->quoted_name;
      copy.basename = partition.basename;
      copy.partitions.clear();
      copy.partitions.emplace_back(partition);

      tasks.emplace_back(std::move(copy));
    }
  }

  for (auto &t : tasks) {
    if (dump) {
      push_table_chunking_task(std::move(t));
    } else {
      push_checksum_task(create_checksum_task(t));
    }
  }
}

void Dumper::push_table_data_task(Table_data_task &&task) {
  ++m_data_tasks_total;

  if (m_checksum) {
    push_checksum_task(create_checksum_task(task));
  }

  std::string info = "dumping " + task.task_name;

  // Table_data_task contains an unique_ptr, it's moved into shared_ptr and
  // then move-captured by lambda, so that this lambda can be stored,
  // otherwise compiler will complain about a call to implicitly-deleted
  // copy constructor
  auto t = std::make_shared<Table_data_task>(std::move(task));

  m_worker_tasks.push({std::move(info),
                       [task = std::move(t)](Table_worker *worker) {
                         worker->dump_table_data(*task);
                       }},
                      shcore::Queue_priority::LOW);
}

void Dumper::push_table_chunking_task(Table_task &&task) {
  ++m_chunking_tasks_total;

  std::string info = "chunking " + task.task_name;
  m_worker_tasks.push({std::move(info),
                       [task = std::move(task)](Table_worker *worker) {
                         worker->create_table_data_tasks(task);
                       }},
                      shcore::Queue_priority::MEDIUM);
}

Dumper::Checksum_task Dumper::create_checksum_task(
    const Table_data_task &table) {
  assert(table.partitions.size() <= 1);

  Checksum_task task;

  task.name = table.task_name;
  task.id = table.id;

  {
    std::lock_guard lock{m_checksums_mutex};
    task.checksum = m_checksum->prepare_checksum(
        table.schema, table.name,
        table.partitions.empty() ? "" : table.partitions.front().info->name,
        table.chunk, table.boundary);
  }

  return task;
}

Dumper::Checksum_task Dumper::create_checksum_task(const Table_task &table) {
  assert(table.partitions.size() <= 1);
  // NOTE: this function is expected to be called from the main thread

  Checksum_task task;

  task.name = table.task_name;
  task.id = table.partitions.empty() ? "whole table" : "whole partition";

  task.checksum = m_checksum->prepare_checksum(
      table.schema, table.name,
      table.partitions.empty() ? "" : table.partitions.front().info->name, -1,
      "");

  return task;
}

void Dumper::push_checksum_task(Checksum_task &&task) {
  ++m_checksum_tasks_total;

  std::string info = "checksumming " + task.name;

  m_worker_tasks.push({std::move(info),
                       [task = std::move(task)](Table_worker *worker) {
                         worker->checksum_table_data(task);
                       }},
                      shcore::Queue_priority::LOWEST);
}

std::unique_ptr<Dumper::Dump_writer_controller> Dumper::table_dump_controller(
    const Output_config *config, const std::string &filename) const {
  if (m_options.use_single_file()) {
    return std::make_unique<Single_file_writer_controller>(
        config->writer_creator(), m_output_file.get());
  } else {
    return std::make_unique<Default_writer_controller>(
        config->writer_creator(),
        [this, config](const std::string &name) {
          return mysqlshdk::storage::make_file(
              make_data_file(config, name, true), config->compression,
              config->compression_options);
        },
        m_options.write_index_files()
            ? [this, config](
                  const std::string
                      &name) { return make_data_file(config, name, false); }
            : Dump_writer_controller::Create_file{},
        filename,
        // We only use the .dumping extension in case of the local files. In
        // case of the remote ones, file is not actually created until the whole
        // data is uploaded, so it's not visible to the loader. This allows to
        // avoid renaming the file, which in some cases is costly.
        m_options.rename_data_files() && config->dir->is_local());
  }
}

std::unique_ptr<Dumper::Dump_writer_controller>
Dumper::table_dump_multi_file_controller(const Output_config *config,
                                         const std::string &basename) const {
  return std::make_unique<Multi_file_writer_controller>(
      [this, config](const std::string &name) {
        return table_dump_controller(config, name);
      },
      basename, config->data_file_extension, m_options.bytes_per_chunk());
}

void Dumper::finish_writing(const std::string &schema, const std::string &table,
                            const Dump_writer_controller *controller) {
  std::lock_guard<std::mutex> lock(m_table_data_stats_mutex);

  controller->update_uncompressed_file_size(&m_chunk_file_bytes);
  m_table_data_stats[schema][table] += controller->total_stats();
}

void Dumper::write_dump_redirected_metadata() const {
  if (m_options.is_export_only()) {
    return;
  }

  shcore::JSON_dumper json{true};

  json.start_object();

  json.append("dumper", std::string("mysqlsh ") + shcore::get_long_version());
  json.append("version", common::k_dumper_version);
  json.append("origin", name());

  {
    // empty list of schemas
    json.append("schemas");
    json.start_array();
    json.end_array();
  }

  {
    // empty map of basenames
    json.append("basenames");
    json.start_object();
    json.end_object();
  }

  append_capability_metadata({Capability::DUMP_DIR_REDIRECTION}, &json);

  json.append("redirectedDumpDir", m_redirected_dump_dir);

  json.end_object();

  common::write_json(make_dump_file(common::k_root_metadata_file), json);
}

void Dumper::write_dump_started_metadata() const {
  if (m_options.is_export_only()) {
    return;
  }

  shcore::JSON_dumper json{true};

  json.start_object();

  json.append("dumper", std::string("mysqlsh ") + shcore::get_long_version());
  json.append("version", common::k_dumper_version);
  json.append("origin", name());

  if (const auto &options = m_options.original_options()) {
    json.append("options", options);
  }

  {
    // list of schemas
    json.append("schemas");
    json.start_array();

    for (const auto &schema : m_schema_infos) {
      json.append(schema.name);
    }

    json.end_array();
  }

  {
    // map of basenames
    json.append("basenames");
    json.start_object();

    for (const auto &schema : m_schema_infos) {
      json.append(schema.name, schema.basename);
    }

    json.end_object();
  }

  if (dump_users()) {
    // list of users
    json.append("users");
    json.start_array();

    for (const auto &user : m_cache.users) {
      json.append(shcore::make_account(user));
    }

    json.end_array();
  }

  if (should_dump_data_masking_policies()) {
    // list of data masking policies
    json.append("dataMaskingPolicies");
    json.start_array();

    for (const auto &policy : m_cache.data_masking_policies) {
      json.append(policy);
    }

    json.end_array();
  }

  json.append("hasMaskedTableDdl", will_dump_masked_table_ddl());
  json.append("maybeHasMaskedTableData",
              will_dump_table_data_with_masked_columns() &&
                  m_options.allow_data_masking());

  json.append("defaultCharacterSet", m_options.character_set());
  json.append("tzUtc", m_options.use_timezone_utc());
  json.append("bytesPerChunk", m_options.bytes_per_chunk());

  json.append("user", m_cache.user);
  json.append("hostname", m_cache.hostname);

  json.append("gtidExecutedInconsistent", is_gtid_executed_inconsistent());
  json.append("consistent", m_options.consistent_dump());

  if (m_options.mds_compatibility()) {
    json.append("mdsCompatibility", true);
  }

  json.append("targetVersion", m_options.target_version().get_full());
  json.append("hasLibraryDdl", m_cache.has_library_ddl);

  json.append("estimatedRowCount", m_total_rows);
  json.append("estimatedDataSize", m_total_data_size);

  {
    // source server
    json.append("source");

    common::serialize(m_cache.server, &json, m_options.dump_binlog_info());
  }

  // TODO(pawel): remove these when dumper's version changes
  {
    json.append("server", m_cache.server.sysvars.hostname);
    json.append("serverVersion", m_cache.server.version.number.get_full());

    if (m_options.dump_binlog_info()) {
      json.append("binlogFile", m_cache.server.binlog.file.name);
      json.append("binlogPosition", m_cache.server.binlog.file.position);
    }

    json.append("gtidExecuted", m_cache.server.binlog.gtid_executed);
    json.append("partialRevokes",
                m_cache.server.sysvars.partial_revokes.value_or(false));
  }

  {
    // list of compatibility options
    json.append("compatibilityOptions");
    json.start_array();

    for (const auto &option : m_options.compatibility_options().values()) {
      json.append(to_string(option));
    }

    json.end_array();
  }

  // list of used capabilities
  append_capability_metadata(m_used_capabilities, &json);

  json.append("checksum", m_options.checksum());

  if (uses_innodb_vector_store()) {
    json.append("hasVectorStoreTables", true);
    json.append("vectorStore");
    common::serialize(m_vector_store_info, &json);
  }

  json.append("begin", m_progress_thread.duration().started_at());

  json.end_object();

  common::write_json(make_dump_file(common::k_root_metadata_file), json);
}

void Dumper::write_dump_finished_metadata() const {
  if (m_options.is_export_only()) {
    return;
  }

  using rapidjson::Document;
  using rapidjson::StringRef;
  using rapidjson::Type;
  using rapidjson::Value;

  Document doc{Type::kObjectType};
  auto &a = doc.GetAllocator();

  doc.AddMember(StringRef("end"),
                {Progress_thread::Duration::current_time().c_str(), a}, a);
  doc.AddMember(StringRef("dataBytes"), m_data_bytes.load(), a);

  {
    Value bytes{Type::kObjectType};
    Value rows{Type::kObjectType};

    for (const auto &schema : m_table_data_stats) {
      Value table_bytes{Type::kObjectType};
      Value table_rows{Type::kObjectType};

      for (const auto &table : schema.second) {
        table_bytes.AddMember(refs(table.first), table.second.data_bytes(), a);
        table_rows.AddMember(refs(table.first), table.second.rows_written(), a);
      }

      bytes.AddMember(refs(schema.first), std::move(table_bytes), a);
      rows.AddMember(refs(schema.first), std::move(table_rows), a);
    }

    doc.AddMember(StringRef("tableDataBytes"), std::move(bytes), a);
    doc.AddMember(StringRef("tableRows"), std::move(rows), a);
  }

  {
    Value files{Type::kObjectType};

    for (const auto &file : m_chunk_file_bytes)
      files.AddMember(refs(file.first), file.second, a);

    doc.AddMember(StringRef("chunkFileBytes"), std::move(files), a);
  }

  {
    Value issues{Type::kObjectType};

    if (m_has_mismatched_view_references) {
      issues.AddMember(StringRef("hasInvalidViewReferences"), true, a);
    }

    if (!issues.ObjectEmpty()) {
      doc.AddMember(StringRef("issues"), std::move(issues), a);
    }
  }

  write_json(make_dump_file(common::k_done_metadata_file), &doc);
}

void Dumper::write_checksum_metadata() const {
  if (!m_checksum) {
    return;
  }

  m_checksum->serialize(make_dump_file(common::k_checksums_metadata_file));
}

void Dumper::write_schema_metadata(
    const Schema_info &schema,
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  if (m_options.is_export_only()) {
    return;
  }

  using rapidjson::Document;
  using rapidjson::StringRef;
  using rapidjson::Type;
  using rapidjson::Value;

  Document doc{Type::kObjectType};
  auto &a = doc.GetAllocator();

  doc.AddMember(StringRef("schema"), refs(schema.name), a);
  doc.AddMember(StringRef("includesDdl"), m_options.dump_ddl(), a);
  doc.AddMember(StringRef("includesViewsDdl"), m_options.dump_ddl(), a);
  doc.AddMember(StringRef("includesData"), m_options.dump_data(), a);

  doc.AddMember(StringRef("multifileDdl"), uses_multifile_schema_ddl(), a);

  {
    // list of tables
    Value tables{Type::kArrayType};

    for (const auto &table : schema.tables) {
      tables.PushBack(refs(table.name), a);
    }

    doc.AddMember(StringRef("tables"), std::move(tables), a);
  }

  if (m_options.dump_ddl()) {
    // list of views
    Value views{Type::kArrayType};

    for (const auto &view : schema.views) {
      views.PushBack(refs(view.name), a);
    }

    doc.AddMember(StringRef("views"), std::move(views), a);
  }

  if (m_options.dump_ddl()) {
    const auto dumper = schema_dumper(session);

    if (m_options.dump_events()) {
      // list of events
      Value events{Type::kArrayType};

      for (const auto &event : dumper->get_events(schema.name)) {
        events.PushBack({event.c_str(), a}, a);
      }

      doc.AddMember(StringRef("events"), std::move(events), a);
    }

    if (m_options.dump_routines()) {
      for (const auto type : {"FUNCTION", "PROCEDURE"}) {
        // list of routines
        Value routines{Type::kArrayType};
        // list of routine dependencies
        Value routine_dependencies{Type::kObjectType};

        for (const auto &routine : dumper->get_routines(schema.name, type)) {
          routines.PushBack({routine.c_str(), a}, a);

          {
            Value libraries{Type::kArrayType};

            for (const auto &dependency :
                 dumper->get_routine_dependencies(schema.name, routine, type)) {
              Value library{Type::kObjectType};

              library.AddMember(StringRef("schema"),
                                {dependency.schema.c_str(), a}, a);
              library.AddMember(StringRef("library"),
                                {dependency.library.c_str(), a}, a);

              libraries.PushBack(std::move(library), a);
            }

            if (libraries.Size()) {
              routine_dependencies.AddMember({routine.c_str(), a},
                                             std::move(libraries), a);
            }
          }
        }

        doc.AddMember({(shcore::str_lower(type) + "s").c_str(), a},
                      std::move(routines), a);
        doc.AddMember({(shcore::str_lower(type) + "Dependencies").c_str(), a},
                      std::move(routine_dependencies), a);
      }
    }

    if (m_options.dump_libraries()) {
      // list of libraries
      Value libraries{Type::kArrayType};

      for (const auto &library : dumper->get_libraries(schema.name)) {
        libraries.PushBack({library.c_str(), a}, a);
      }

      doc.AddMember(StringRef("libraries"), std::move(libraries), a);
    }
  }

  {
    // map of basenames
    Value basenames{Type::kObjectType};

    for (const auto &table : schema.tables) {
      basenames.AddMember(refs(table.name), refs(table.basename), a);
    }

    for (const auto &view : schema.views) {
      basenames.AddMember(refs(view.name), refs(view.basename), a);
    }

    doc.AddMember(StringRef("basenames"), std::move(basenames), a);
  }

  write_json(
      make_dump_file(common::get_schema_filename(schema.basename, "json")),
      &doc);
}

void Dumper::write_table_metadata(
    const Table_task &table,
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  if (m_options.is_export_only()) {
    return;
  }

  using rapidjson::Document;
  using rapidjson::StringRef;
  using rapidjson::Type;
  using rapidjson::Value;

  Document doc{Type::kObjectType};
  auto &a = doc.GetAllocator();

  {
    // options - to be used by importer
    Value options{Type::kObjectType};

    options.AddMember(StringRef("schema"), refs(table.schema), a);
    options.AddMember(StringRef("table"), refs(table.name), a);

    Value cols{Type::kArrayType};
    Value decode{Type::kObjectType};

    for (const auto &c : table.info->columns) {
      cols.PushBack(refs(c->name), a);

      if (const auto decoded = decode_column(table.info, c); !decoded.empty()) {
        decode.AddMember(refs(c->name), {decoded.c_str(), a}, a);
      }
    }

    options.AddMember(StringRef("columns"), std::move(cols), a);

    if (!decode.ObjectEmpty()) {
      options.AddMember(StringRef("decodeColumns"), std::move(decode), a);
    }

    options.AddMember(StringRef("defaultCharacterSet"),
                      refs(m_options.character_set()), a);

    options.AddMember(StringRef("fieldsTerminatedBy"),
                      refs(table.output_config->dialect.fields_terminated_by),
                      a);
    options.AddMember(StringRef("fieldsEnclosedBy"),
                      refs(table.output_config->dialect.fields_enclosed_by), a);
    options.AddMember(StringRef("fieldsOptionallyEnclosed"),
                      table.output_config->dialect.fields_optionally_enclosed,
                      a);
    options.AddMember(StringRef("fieldsEscapedBy"),
                      refs(table.output_config->dialect.fields_escaped_by), a);
    options.AddMember(StringRef("linesTerminatedBy"),
                      refs(table.output_config->dialect.lines_terminated_by),
                      a);

    doc.AddMember(StringRef("options"), std::move(options), a);
  }

  const auto dumper = schema_dumper(session);

  if (m_options.dump_triggers() && m_options.dump_ddl()) {
    // list of triggers
    Value triggers{Type::kArrayType};

    for (const auto &trigger : dumper->get_triggers(table.schema, table.name)) {
      triggers.PushBack({trigger.c_str(), a}, a);
    }

    doc.AddMember(StringRef("triggers"), std::move(triggers), a);
  }

  if (const auto &all_histograms =
          dumper->get_histograms(table.schema, table.name);
      !all_histograms.empty()) {
    // list of histograms
    Value histograms{Type::kArrayType};

    for (const auto &histogram : all_histograms) {
      Value h{Type::kObjectType};

      h.AddMember(StringRef("column"), refs(histogram.column), a);
      h.AddMember(StringRef("buckets"),
                  static_cast<uint64_t>(histogram.buckets), a);

      histograms.PushBack(std::move(h), a);
    }

    doc.AddMember(StringRef("histograms"), std::move(histograms), a);
  }

  doc.AddMember(StringRef("includesData"),
                m_options.dump_data() && should_dump_data(table), a);
  doc.AddMember(StringRef("includesDdl"), m_options.dump_ddl(), a);

  doc.AddMember(StringRef("extension"),
                refs(table.output_config->data_file_extension), a);
  doc.AddMember(StringRef("chunking"), m_options.split(), a);
  doc.AddMember(
      StringRef("compression"),
      {mysqlshdk::storage::to_string(table.output_config->compression).c_str(),
       a},
      a);

  {
    Value primary{Type::kArrayType};

    if (table.info->primary_key) {
      for (const auto &column : table.info->primary_key->columns()) {
        primary.PushBack(refs(column->name), a);
      }
    }

    doc.AddMember(StringRef("primaryIndex"), std::move(primary), a);
    doc.AddMember(StringRef("hasPKE"),
                  !table.info->primary_key_equivalents.empty(), a);
  }

  {
    // list of partitions
    Value partitions{Type::kArrayType};

    for (const auto &partition : table.partitions) {
      partitions.PushBack(refs(partition.info->name), a);
    }

    doc.AddMember(StringRef("partitions"), std::move(partitions), a);
  }

  {
    // map of partition basenames
    Value basenames{Type::kObjectType};

    for (const auto &partition : table.partitions) {
      basenames.AddMember(refs(partition.info->name), refs(partition.basename),
                          a);
    }

    doc.AddMember(StringRef("basenames"), std::move(basenames), a);
  }

  {
    const auto &where = m_options.where(table.schema, table.name);

    if (!where.empty()) {
      doc.AddMember(StringRef("where"), refs(where), a);
    }
  }

  if (uses_innodb_vector_store() && table.info->is_innodb_vector_store_table) {
    doc.AddMember(StringRef("isVectorStoreTable"), true, a);
  }

  if (uses_dynamic_data_masking()) {
    doc.AddMember(StringRef("hasMaskingPolicy"), table.info->has_masking_policy,
                  a);
  }

  write_json(
      make_dump_file(common::get_table_data_filename(table.basename, "json")),
      &doc);
}

void Dumper::summarize() const {
  const auto console = current_console();

  if (!m_options.consistent_dump() && m_options.threads() > 1) {
    const auto &prev_gtid_executed = m_cache.server.binlog.gtid_executed;
    const auto gtid_executed = this->gtid_executed(session());

    if (gtid_executed != prev_gtid_executed) {
      console->print_warning(
          "The value of the gtid_executed system variable has changed during "
          "the dump, from: '" +
          prev_gtid_executed + "' to: '" + gtid_executed + "'.");
    }
  }

  if (m_cache.has_library_ddl &&
      !compatibility::supports_library_ddl(m_options.target_version())) {
    console->print_warning(shcore::str_format(
        "The dump contains library DDL, however the 'targetVersion' option is "
        "set to %s, where this feature is not supported.",
        m_options.target_version().get_base().c_str()));
  }

  if (uses_innodb_vector_store() &&
      !compatibility::supports_vector_store_conversion(
          m_options.target_version())) {
    console->print_warning(shcore::str_format(
        "The dump contains InnoDB-based vector store tables, however the "
        "'targetVersion' option is set to %s, where automatic conversion to "
        "Lakehouse is not supported.",
        m_options.target_version().get_base().c_str()));
  }

  if (uses_dynamic_data_masking() &&
      !compatibility::supports_dynamic_data_masking(
          m_options.target_version())) {
    // WL17279-FR1.4.1: warn if `targetVersion` doesn't support dynamic data
    // masking
    console->print_warning(
        shcore::str_format("The dump contains dynamic data masking DDL, "
                           "however the 'targetVersion' option is set to %s, "
                           "where this feature is not supported.",
                           m_options.target_version().get_base().c_str()));
  }

  if (uses_dump_dir_redirection()) {
    console->print_status("Main dump directory: " +
                          m_options.describe(m_options.output_url()));
    console->print_status("Dump location: " +
                          m_options.describe(m_redirected_output_url));
  }

  if (uses_innodb_vector_store()) {
    // WL16802-FR1.1.2: print location of the vector store data
    console->print_status("Vector store location: " +
                          m_vector_store_output_description);
  }

  if (m_options.dump_data()) {
    console->print_status("Dump duration: " +
                          m_data_dump_stage->duration().to_string());
  }

  if (m_options.checksum()) {
    console->print_status("Checksum duration: " +
                          m_checksum_duration.to_string());
  }

  console->print_status("Total duration: " +
                        m_progress_thread.duration().to_string());

  if (!m_options.is_export_only()) {
    console->print_status("Schemas dumped: " + std::to_string(m_total_schemas));

    {
      std::string msg = "Tables dumped: ";
      msg += std::to_string(m_total_tables);

      if (uses_innodb_vector_store()) {
        msg += ", including ";
        msg += std::to_string(m_cache.innodb_vector_store_tables);
        msg += " InnoDB-based vector store tables";
      }

      console->print_status(msg);
    }
  }

  if (m_options.dump_data()) {
    console->print_status(
        std::string(compressed() ? "Uncompressed d" : "D") +
        "ata size: " + mysqlshdk::utils::format_bytes(m_data_bytes));

    if (compressed()) {
      console->print_status("Compressed data size: " +
                            mysqlshdk::utils::format_bytes(m_bytes_written));
      console->print_status(shcore::str_format(
          "Compression ratio: %.1f",
          m_data_bytes / std::max(static_cast<double>(m_bytes_written), 1.0)));
    }

    console->print_status("Rows written: " + std::to_string(m_rows_written));
    console->print_status("Bytes written: " +
                          mysqlshdk::utils::format_bytes(m_bytes_written));
    console->print_status(
        std::string("Average ") + (compressed() ? "uncompressed " : "") +
        "throughput: " +
        mysqlshdk::utils::format_throughput_bytes(
            m_data_bytes, m_data_dump_stage->duration().seconds()));

    if (compressed()) {
      console->print_status(
          "Average compressed throughput: " +
          mysqlshdk::utils::format_throughput_bytes(
              m_bytes_written, m_data_dump_stage->duration().seconds()));
    }
  }

  summary();
}

void Dumper::rethrow() const {
  bool failure = false;

  for (std::size_t i = 0, e = m_worker_exceptions.size(); i < e; ++i) {
    if (m_worker_exceptions[i]) {
      dump::log_exception(shcore::str_format("%s() (thread %zu)", name(), i),
                          m_worker_exceptions[i]);
      failure = true;
    }
  }

  if (failure) {
    THROW_ERROR(SHERR_DUMP_WORKER_THREAD_FATAL_ERROR);
  }
}

void Dumper::emergency_shutdown() {
  m_worker_interrupt.test_and_set();

  if (const auto workers = m_workers.size()) {
    m_worker_tasks.shutdown(workers);
  }
}

void Dumper::kill_workers() {
  emergency_shutdown();
  wait_for_all_tasks();
}

void Dumper::initialize_throughput_progress() {
  if (!m_options.dump_data()) {
    return;
  }

  Progress_thread::Throughput_config config;

  config.items_full = "rows";
  config.items_abbrev = "rows";
  config.item_singular = "row";
  config.item_plural = "rows";
  config.space_before_item = true;
  config.total_is_approx = true;
  config.current = [this]() -> uint64_t { return m_rows_written; };
  config.total = [this]() { return m_total_rows; };

  if (!m_options.is_export_only()) {
    config.left_label = [this]() {
      std::string label;

      if (const auto chunking = m_num_threads_chunking.load()) {
        label += std::to_string(chunking) + " thds chunking - ";
      }

      if (const auto dumping = m_num_threads_dumping.load()) {
        label += std::to_string(dumping) + " thds dumping - ";
      }

      if (const auto checksumming = m_num_threads_checksumming.load()) {
        label += std::to_string(checksumming) + " thds chksum - ";
      }

      constexpr std::array k_progress_spin = {'-', '\\', '|', '/'};
      static thread_local size_t progress_idx = 0;

      if (label.length() > 2) {
        label[label.length() - 2] = k_progress_spin[progress_idx++];

        if (progress_idx >= k_progress_spin.size()) {
          progress_idx = 0;
        }
      }

      return label;
    };
  }

  if (m_progress_thread.uses_json_output()) {
    config.extra_attributes = [this]() {
      auto dump_progress = shcore::make_dict();

      {
        std::lock_guard<std::recursive_mutex> lock(m_throughput_mutex);

        {
          const auto rate = m_data_throughput->rate();
          const double seconds = 1.0;
          dump_progress->emplace(
              "dataThroughput",
              mysqlshdk::textui::format_json_throughput(rate, seconds));
        }

        if (compressed()) {
          {
            const auto rate = m_bytes_throughput->rate();
            const double seconds = 1.0;
            dump_progress->emplace(
                "fileThroughput",
                mysqlshdk::textui::format_json_throughput(rate, seconds));
          }
        }
      }

      return shcore::make_dict("dumpProgress", std::move(dump_progress));
    };
  }

  config.right_label = [this]() { return ", " + throughput(); };
  config.on_display_started = []() {
    current_console()->print_status("Starting data dump");
  };

  m_data_dump_stage =
      m_progress_thread.start_stage("Dumping data", std::move(config));
}

void Dumper::initialize_checksum_progress() {
  if (!m_options.checksum()) {
    return;
  }

  dump::Progress_thread::Progress_config config;
  config.current = [this]() { return m_checksum_tasks_completed.load(); };
  config.total = [this]() { return m_checksum_tasks_total.load(); };
  config.is_total_known = [this]() { return all_tasks_produced(); };

  m_progress_thread.start_stage({"Computing checksum", false},
                                std::move(config));
}

void Dumper::update_progress(const Dump_write_result &progress) {
  m_rows_written += progress.rows_written();
  m_bytes_written += progress.bytes_written();
  m_data_bytes += progress.data_bytes();

  {
    std::unique_lock<std::recursive_mutex> lock(m_throughput_mutex,
                                                std::try_to_lock);
    if (lock.owns_lock()) {
      m_data_throughput->push(m_data_bytes);
      m_bytes_throughput->push(m_bytes_written);
    }
  }
}

void Dumper::shutdown_progress() { m_progress_thread.finish(); }

std::string Dumper::throughput() const {
  std::lock_guard<std::recursive_mutex> lock(m_throughput_mutex);

  return mysqlshdk::utils::format_throughput_bytes(m_data_throughput->rate(),
                                                   1.0) +
         (compressed() ? " uncompressed, " +
                             mysqlshdk::utils::format_throughput_bytes(
                                 m_bytes_throughput->rate(), 1.0) +
                             " compressed"
                       : "");
}

std::unique_ptr<mysqlshdk::storage::IFile> Dumper::make_dump_file(
    const std::string &filename) const {
  return dump_dir()->file(filename);
}

std::unique_ptr<mysqlshdk::storage::IFile> Dumper::make_data_file(
    const Output_config *config, const std::string &filename,
    bool use_mmap) const {
  const auto &file_options =
      use_mmap ? m_mmap_options : mysqlshdk::storage::File_options{};
  return config->dir->file(filename, file_options);
}

std::string Dumper::get_basename(const std::string &basename) {
  return m_basenames.get(basename);
}

bool Dumper::compressed() const {
  return mysqlshdk::storage::Compression::NONE != m_options.compression();
}

std::string Dumper::get_query_comment(const std::string &quoted_name,
                                      const std::string &id,
                                      const char *context) const {
  return "/* mysqlsh " + mysqlsh::common::member_name(name()) + ", " + context +
         " table " +
         // sanitize schema/table names in case they contain a '*/'
         // *\/ isn't really a valid escape, but it doesn't matter because we
         // just want the lexer to not see */
         shcore::str_replace(quoted_name, "*/", "*\\/") + ", ID: " + id + " */";
}

std::string Dumper::get_query_comment(const Table_data_task &task,
                                      const char *context) const {
  return get_query_comment(task.task_name, task.id, context);
}

bool Dumper::should_dump_data(const Table_task &table) const {
  if (table.info->columns.empty() ||
      (table.schema == "mysql" &&
       (table.name == "apply_status" || table.name == "general_log" ||
        table.name == "schema" || table.name == "slow_log"))) {
    return false;
  } else {
    return true;
  }
}

void Dumper::validate_privileges() const {
  std::set<std::string> all_required;
  std::set<std::string> global_required;
  std::set<std::string> schema_required;
  std::set<std::string> table_required;

  if (m_options.dump_events()) {
    std::string event{"EVENT"};
    all_required.emplace(event);
    schema_required.emplace(std::move(event));
  }

  if (m_options.dump_triggers()) {
    std::string trigger{"TRIGGER"};
    all_required.emplace(trigger);
    table_required.emplace(std::move(trigger));
  }

  if (!m_cache.server.version.is_8_0) {
    // need to explicitly check for SELECT privilege, otherwise some queries
    // will return empty results
    std::string select{"SELECT"};
    all_required.emplace(select);
    table_required.emplace(std::move(select));
  }

  if (m_cache.server.version.is_5_6 && dump_users()) {
    // on 5.6, SUPER is required to get the real hash value from SHOW GRANTS
    std::string super{"SUPER"};
    all_required.emplace(super);
    global_required.emplace(std::move(super));
  }

  // WL17279-FR1.3: MANAGE_DATA_MASKING_POLICY privilege is needed when dumping
  // data masking policies
  if (should_dump_data_masking_policies()) {
    std::string manage_data_masking_policy{"MANAGE_DATA_MASKING_POLICY"};
    all_required.emplace(manage_data_masking_policy);
    global_required.emplace(std::move(manage_data_masking_policy));
  }

  if (!all_required.empty()) {
    using mysqlshdk::mysql::User_privileges_result;

    const auto get_missing = [](const User_privileges_result &result,
                                const std::set<std::string> &required) {
      std::set<std::string> missing;

      std::set_intersection(result.missing_privileges().begin(),
                            result.missing_privileges().end(), required.begin(),
                            required.end(),
                            std::inserter(missing, missing.begin()));

      return missing;
    };

    const auto global_result = m_user_privileges->validate(all_required);
    const auto global_missing = get_missing(global_result, global_required);

    if (!global_missing.empty()) {
      THROW_ERROR(SHERR_DUMP_MISSING_GLOBAL_PRIVILEGES,
                  shcore::make_account(m_user_account).c_str(),
                  shcore::str_join(global_missing, ", ").c_str());
    }

    if (global_result.has_missing_privileges()) {
      {
        // global privileges can be safely removed from the all_required set
        std::set<std::string> temporary;

        std::set_difference(all_required.begin(), all_required.end(),
                            global_required.begin(), global_required.end(),
                            std::inserter(temporary, temporary.begin()));

        all_required = std::move(temporary);
      }

      // user has all required global privileges
      // user doesn't have *.* schema/table-level privileges, check schemas
      for (const auto &schema : m_schema_infos) {
        const auto schema_result =
            m_user_privileges->validate(all_required, schema.name);
        const auto schema_missing = get_missing(schema_result, schema_required);

        if (!schema_missing.empty()) {
          THROW_ERROR(SHERR_DUMP_MISSING_SCHEMA_PRIVILEGES,
                      shcore::make_account(m_user_account).c_str(),
                      schema.quoted_name.c_str(),
                      shcore::str_join(schema_missing, ", ").c_str());
        }

        if (schema_result.has_missing_privileges()) {
          // user has all required schema-level privileges for this schema
          // user doesn't have schema.* table-level privileges, check tables
          for (const auto &table : schema.tables) {
            const auto table_result = m_user_privileges->validate(
                all_required, schema.name, table.name);

            // if at this stage there are any missing privileges, they are all
            // table-level ones
            if (table_result.has_missing_privileges()) {
              THROW_ERROR(
                  SHERR_DUMP_MISSING_TABLE_PRIVILEGES,
                  shcore::make_account(m_user_account).c_str(),
                  table.quoted_name.c_str(),
                  shcore::str_join(table_result.missing_privileges(), ", ")
                      .c_str());
            }
          }
        }
      }
    }
  }
}

bool Dumper::is_gtid_executed_inconsistent() const {
  return !m_options.consistent_dump();
}

void Dumper::validate_schemas_list() const {
  if (!dump_users() && m_cache.schemas.empty()) {
    THROW_ERROR(SHERR_DUMP_NO_SCHEMAS_SELECTED);
  }
}

void Dumper::log_server_version() const {
  log_info("Source server: %s",
           query("SELECT CONCAT(@@version, ' ', @@version_comment)")
               ->fetch_one()
               ->get_string(0)
               .c_str());
}

void Dumper::print_object_stats() const {
  auto stats = object_stats(m_cache.filtered, m_cache.total);

  if (stats.empty()) {
    return;
  }

#define FORMAT_OBJECT_STATS(type)                                         \
  do {                                                                    \
    if (m_options.dump_##type() && 0 != m_cache.total.type) {             \
      stats.emplace_back(format_object_stats(m_cache.filtered.type,       \
                                             m_cache.total.type, #type)); \
    }                                                                     \
  } while (false)

  if (m_options.dump_ddl()) {
    FORMAT_OBJECT_STATS(events);
    FORMAT_OBJECT_STATS(libraries);
    FORMAT_OBJECT_STATS(routines);
    FORMAT_OBJECT_STATS(triggers);
  }

#undef FORMAT_OBJECT_STATS

  const auto console = current_console();

  {
    std::string msg = stats.front() + " will be dumped";

    auto current = std::next(stats.begin());
    const auto end = stats.end();

    if (end != current) {
      msg += " and within them";
    } else {
      msg += '.';
    }

    for (; end != current; ++current) {
      msg += ' ' + (*current) + ',';
    }

    msg.back() = '.';

    console->print_status(msg);
  }

  if (m_options.dump_users()) {
    if (m_skip_grant_tables_active) {
      console->print_warning(
          "Server is running with the --skip-grant-tables option active, "
          "dumping users, roles and grants has been disabled.");
    } else {
      console->print_status(format_object_stats(m_cache.filtered.users,
                                                m_cache.total.users, "users") +
                            " will be dumped.");
    }
  }

  if (should_dump_data_masking_policies()) {
    const auto count = m_cache.data_masking_policies.size();

    console->print_status(
        format_object_stats(count, count, "data masking policies") +
        " will be dumped.");
  }
}

std::string Dumper::format_object_stats(uint64_t value, uint64_t total,
                                        std::string_view objects) {
  std::string format;

  if (value != total) {
    format += std::to_string(value);
    format += " out of ";
  }

  format += std::to_string(total);
  format += ' ';

  bool append_objects = true;

  if (1 == total) {
    if (objects.ends_with("ies")) {
      objects.remove_suffix(3);

      format += objects;
      format += 'y';

      append_objects = false;
    } else if ('s' == objects.back()) {
      objects.remove_suffix(1);
    }
  }

  if (append_objects) {
    format += objects;
  }

  return format;
}

bool Dumper::dump_users() const {
  return m_options.dump_users() && !m_skip_grant_tables_active;
}

void Dumper::validate_dump_consistency(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  if (!m_options.consistent_dump() || m_ftwrl_used || m_user_has_backup_admin) {
    return;
  }

  bool consistent = false;
  const auto instance = mysqlshdk::mysql::Instance(session);
  const auto console = current_console();
  const auto skip_check = [&console, &consistent]() {
    console->print_note(
        "The 'skipConsistencyChecks' option is set, assuming there were no DDL "
        "changes.");
    consistent = true;
  };

  if (m_gtid_enabled) {
    const auto &prev_gtid_executed = m_cache.server.binlog.gtid_executed;
    const auto gtid_executed = this->gtid_executed(session);
    consistent = gtid_executed == prev_gtid_executed;

    if (!consistent) {
      console->print_note(
          "The value of the gtid_executed system variable has changed during "
          "the dump, from: '" +
          prev_gtid_executed + "' to: '" + gtid_executed + "'.");

      if (m_options.skip_consistency_checks()) {
        skip_check();
      } else {
        // check if executed statements are safe
        // get GTID sets which were executed since the dump has started

        const auto set =
            Gtid_set::from_normalized_string(gtid_executed)
                .subtract(Gtid_set::from_normalized_string(prev_gtid_executed),
                          instance);

        consistent = check_if_transactions_are_ddl_safe(
            instance, m_cache.server.binlog.file, binlog(session).file, set);
      }
    }
  } else {
    // gtid_mode is not enabled, check binlog
    const auto binlog = this->binlog(session).file;
    consistent = m_cache.server.binlog.file == binlog;

    if (!consistent) {
      console->print_note(
          "The binlog position has changed during the dump, from: '" +
          m_cache.server.binlog.file.to_string() + "' to: '" +
          binlog.to_string() + "'.");

      if (m_options.skip_consistency_checks()) {
        skip_check();
      } else {
        // check if executed statements are safe
        consistent = check_if_transactions_are_ddl_safe(
            instance, m_cache.server.binlog.file, binlog);
      }
    }
  }

  auto msg = "Backup lock is not " + why_backup_lock_is_missing() +
             " and DDL changes were not blocked.";

  if (!consistent) {
    msg += " The consistency of the dump cannot be guaranteed.";

    if (shcore::str_caseeq(name(), "dumpInstance")) {
      console->print_error(msg);
      console->print_note(
          "In order to overcome this issue, use a read-only replica with "
          "replication stopped, or, if dumping from a primary, then enable "
          "super_read_only system variable and ensure that any inbound "
          "replication channels are stopped.");
      THROW_ERROR(SHERR_DUMP_CONSISTENCY_CHECK_FAILED);
    } else {
      console->print_warning(msg);
    }
  } else {
    msg += " The DDL is consistent, the world may resume now.";
    console->print_note(msg);
  }
}

void Dumper::fetch_server_information() {
  const auto instance = mysqlshdk::mysql::Instance(session());

  m_server_version = common::server_version(session());
  m_binlog_enabled = instance.get_sysvar_bool("log_bin").value_or(false);
  m_gtid_enabled = shcore::str_ibeginswith(
      instance.get_sysvar_string("gtid_mode").value_or("OFF"), "ON");

  DBUG_EXECUTE_IF("dumper_binlog_disabled", { m_binlog_enabled = false; });
  DBUG_EXECUTE_IF("dumper_gtid_disabled", { m_gtid_enabled = false; });

  // BUG#37866205 disallow dumps from server with a greater major version
  DBUG_EXECUTE_IF("dumper_unsupported_server_version", {
    m_server_version.number = Version(k_shell_version.get_major() + 1, 0, 0);
  });

  if (m_server_version.number.get_major() > k_shell_version.get_major()) {
    throw std::runtime_error(
        shcore::str_format("Unsupported MySQL Server %s detected, please "
                           "upgrade the MySQL Shell first",
                           m_server_version.number.get_base().c_str()));
  }

  // BUG#36701854 given that minor versions may introduce breaking changes, we
  // warn if server has a greater minor version (8.0.x -> 8.1.0, 8.4.0 -> 8.5.0
  // -> does not exist, but handles also 9.0.0)
  const auto newer_version =
      Version{k_shell_version.get_major(), k_shell_version.get_minor() + 1, 0};

  DBUG_EXECUTE_IF("dumper_newer_server_version",
                  { m_server_version.number = newer_version; });

  if (m_server_version.number >= newer_version) {
    current_console()->print_warning(shcore::str_format(
        "MySQL Server %s detected, which is newer than the MySQL Shell. Please "
        "upgrade the MySQL Shell if dump or load operation fails.",
        m_server_version.number.get_base().c_str()));
  }

  if (compatibility::supports_dynamic_data_masking(m_server_version.number)) {
    m_data_masking_enabled =
        mysqlsh::common::Data_masking{session()}.is_component_installed();
  }
}

issues::Status_set Dumper::check_for_upgrade_errors() const {
  if (!m_options.mds_compatibility()) {
    return {};
  }

  if (m_options.skip_upgrade_checks()) {
    current_console()->print_note("Skipping upgrade compatibility checks");
    return {};
  }

  // upgrade checker does not support 5.6 servers
  if (m_cache.server.version.is_5_6) {
    current_console()->print_note(
        shcore::str_format("MySQL Server %s is not supported, skipping upgrade "
                           "compatibility checks",
                           m_cache.server.version.number.get_full().c_str()));
    return {};
  }

  if (m_options.target_version() <= m_cache.server.version.number) {
    current_console()->print_note(shcore::str_format(
        "The value of 'targetVersion' option (%s) is not greater than current "
        "version of the server (%s), skipping upgrade compatibility checks",
        m_options.target_version().get_base().c_str(),
        m_cache.server.version.number.get_base().c_str()));
    return {};
  }

  if (m_cache.server.version.number == k_shell_version) {
    current_console()->print_note(
        shcore::str_format("Source MySQL Server %s has the same version as the "
                           "MySQL Shell, skipping upgrade compatibility checks",
                           m_cache.server.version.number.get_base().c_str()));
    return {};
  }

  if (m_cache.server.version.number > k_shell_version) {
    current_console()->print_note(
        shcore::str_format("Source MySQL Server %s is newer than the MySQL "
                           "Shell, skipping upgrade compatibility checks",
                           m_cache.server.version.number.get_base().c_str()));
    return {};
  }

  upgrade_checker::Upgrade_check_options options;
  options.target_version = m_options.target_version();
  // It is pointless to perform this check should not be executed in the context
  // of D&L since the mysql schema is excluded all the time.
  options.exclude_list.emplace(upgrade_checker::ids::k_mysql_schema_check);

  if (m_options.compatibility_options().is_set(
          Compatibility_option::FORCE_NON_STANDARD_FKS)) {
    // exclude the non-standard FKs check
    options.exclude_list.emplace(
        upgrade_checker::ids::k_foreign_key_references);
  }

  // we did our own target version checks, and we're less strict than upgrade
  // checker, skip its checks
  options.skip_target_version_check = true;

  issues::Status_set status;
  upgrade_checker::Upgrade_check_config config{options};

  config.set_session(session());
  config.set_user_privileges(m_user_privileges.get());
  // we check engines even though MDS forces InnoDB, because "force_innodb"
  // compatibility option is not going to be able to fix them
  config.set_targets(upgrade_checker::Target_flags(
                         upgrade_checker::Target::OBJECT_DEFINITIONS) |
                     upgrade_checker::Target::ENGINES |
                     upgrade_checker::Target::MDS_SPECIFIC);

  config.set_issue_filter([&status](
                              const upgrade_checker::Upgrade_issue &issue) {
    if (upgrade_checker::ids::k_foreign_key_references == issue.check_name) {
      status.set(issues::Status::ERROR_HAS_NON_STANDARD_FKS);
      status.set(issues::Status::ERROR);
    }

    // we're using DB filters, issues are already filtered
    return true;
  });
  config.set_db_filters(&m_options.filters());

  config.set_warn_on_excludes(false);

  if (!check_for_upgrade(config)) {
    status.set(issues::Status::ERROR);
  }

  return status;
}

void Dumper::throw_if_cannot_dump_users() const {
  if (m_server_version.is_maria_db && dump_users()) {
    THROW_ERROR(SHERR_DUMP_USERS_MARIA_DB_NOT_SUPPORTED);
  }
}

bool Dumper::all_tasks_produced() const {
  // chunking tasks produce data and checksum tasks, if these are finished, then
  // all other tasks were produced
  return m_main_thread_finished_producing_chunking_tasks &&
         m_chunking_tasks_completed == m_chunking_tasks_total;
}

void Dumper::handle_mismatched_view_references(issues::Status_set status,
                                               bool as_error) const {
  if (!status.is_set(issues::Status::WARNING_HAS_MISMATCHED_VIEW_REFERENCES)) {
    return;
  }

  bool expected = false;

  if (!m_has_mismatched_view_references.compare_exchange_strong(expected,
                                                                true)) {
    return;
  }

  const auto console = current_console();

  console->print_info();
  std::invoke(as_error ? &IConsole::print_error : &IConsole::print_note,
              console,
              "One or more views that reference tables using the wrong case "
              "were found.",
              IConsole::Json_attributes{});
  console->print_info(
      "Loading them in a system that uses lower_case_table_names=0 (such as "
      "in the MySQL HeatWave Service) will fail unless they are fixed.");
}

std::string Dumper::gtid_executed(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  return common::gtid_executed(session);
}

common::Binlog Dumper::binlog(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  return common::binlog(session, m_server_version, true);
}

std::string Dumper::optimizer_hints(const Instance_cache::Table *info) const {
  if (!m_server_version.is_8_0) {
    return "SQL_NO_CACHE ";
  } else if ("RAPID" == info->secondary_engine) {
    // disable off-loading to heatwave
    std::string hint = "/*+ SET_VAR(use_secondary_engine='OFF') ";

    // disable hypergraph optimizer as a workaround for BUG#35443375
    if (const auto version = m_server_version.number.numeric();
        version >= 80400 && version < 90000) {
      hint += "SET_VAR(optimizer_switch='hypergraph_optimizer=OFF') ";
    }

    hint += "*/ ";

    return hint;
  } else {
    return {};
  }
}

Dump_writer::Encoding_type Dumper::encoding_type(
    const Instance_cache::Table *table,
    const Instance_cache::Column *column) const {
  if (column->csv_unsafe) {
    // due to BUG#38166093, storing a vector column using VECTOR_TO_STRING() may
    // lead to a loss of precision and cause checksum errors, we use Base64
    // instead
    if (uses_innodb_vector_store() && table->is_innodb_vector_store_table &&
        column->is_innodb_vector_store_column) {
      return Dump_writer::Encoding_type::BASE64;
    } else {
      return m_options.use_base64() ? Dump_writer::Encoding_type::BASE64
                                    : Dump_writer::Encoding_type::HEX;
    }
  } else {
    return Dump_writer::Encoding_type::NONE;
  }
}

std::string Dumper::decode_column(const Instance_cache::Table *table,
                                  const Instance_cache::Column *column) const {
  switch (encoding_type(table, column)) {
    case Dump_writer::Encoding_type::BASE64:
      return "FROM_BASE64";

    case Dump_writer::Encoding_type::HEX:
      return "UNHEX";

    case Dump_writer::Encoding_type::VECTOR:
      // TODO(pawel): just 'STRING_TO_VECTOR' once dumper's version changes
      return shcore::str_format("STRING_TO_VECTOR(@%s)",
                                column->quoted_name.c_str());

    case Dump_writer::Encoding_type::NONE:
      // no need to decode the column
      return {};
  }

  throw std::logic_error{"Dumper::decode_column()"};
}

void Dumper::maybe_ddl_dump_finished(
    const std::shared_ptr<mysqlshdk::db::ISession> &session) const {
  // check if there are still metadata or DDL files to be written
  if (!(m_table_metadata_to_write == m_table_metadata_written &&
        m_schema_metadata_to_write == m_schema_metadata_written &&
        m_ddl_to_write == m_ddl_written)) {
    return;
  }

  validate_compatibility_check_status();
  validate_dump_consistency(session);
}

void Dumper::validate_compatibility_check_status() const {
  std::lock_guard lock{m_compatibility_status_mutex};

  if (m_compatibility_status.is_set(issues::Status::ERROR_HAS_INVALID_GRANTS)) {
    THROW_ERROR(SHERR_DUMP_INVALID_GRANT_STATEMENT);
  } else if (m_compatibility_status.is_set(issues::Status::ERROR)) {
    THROW_ERROR(SHERR_DUMP_COMPATIBILITY_ISSUES_FOUND);
  }
}

bool Dumper::should_write_metadata() const noexcept {
  return Dry_run::DONT_WRITE_ANY_FILES != m_options.dry_run_mode() &&
         !m_options.is_export_only();
}

issues::Status_set Dumper::show_issues(
    const std::vector<Compatibility_issue> &issues) const {
  const auto console = current_console();
  issues::Status_set status;

  const auto issue_attribs = [](const Compatibility_issue &issue) {
    auto ocimds_issue = shcore::make_dict();
    ocimds_issue->emplace("check", to_string(issue.check));
    ocimds_issue->emplace("status", to_string(issue.status));
    ocimds_issue->emplace("objectType", to_string(issue.object_type));
    ocimds_issue->emplace("objectName", issue.object_name);
    ocimds_issue->emplace("description", issue.description);

    if (!issue.compatibility_options.empty()) {
      auto array = shcore::make_array();

      for (auto option : issue.compatibility_options.values()) {
        array->emplace_back(to_string(option));
      }

      ocimds_issue->emplace("compatibilityOptions", std::move(array));
    }

    return shcore::make_dict("compatibilityIssue", std::move(ocimds_issue));
  };

  for (const auto &issue : issues) {
    switch (issue.status) {
      case Compatibility_issue::Status::FIXED: {
        status.set(issues::Status::FIXED);

        if (Compatibility_check::TABLE_MISSING_PK == issue.check) {
          if (issue.compatibility_options.is_set(
                  Compatibility_option::CREATE_INVISIBLE_PKS)) {
            status.set(issues::Status::FIXED_CREATE_PKS);
          } else if (issue.compatibility_options.is_set(
                         Compatibility_option::IGNORE_MISSING_PKS)) {
            status.set(issues::Status::FIXED_IGNORE_PKS);
          }
        }

        console->print_note(issue.description, issue_attribs(issue));
        break;
      }

      case Compatibility_issue::Status::NOTE: {
        console->print_note(issue.description, issue_attribs(issue));
        break;
      }

      case Compatibility_issue::Status::WARNING: {
        status.set(issues::Status::WARNING);

        bool show_issue = true;

        if (Compatibility_check::OBJECT_INVALID_DEFINER == issue.check ||
            Compatibility_check::OBJECT_MISSING_SQL_SECURITY == issue.check) {
          status.set(issues::Status::WARNING_DEPRECATED_DEFINERS);
        } else if (Compatibility_check::USER_ESCAPED_WILDCARD_GRANT ==
                   issue.check) {
          status.set(issues::Status::WARNING_ESCAPED_WILDCARDS);
        } else if (Compatibility_check::VIEW_MISMATCHED_REFERENCE ==
                   issue.check) {
          status.set(issues::Status::WARNING_HAS_MISMATCHED_VIEW_REFERENCES);
        } else if (Compatibility_check::
                       OBJECT_INVALID_DEFINER_USERS_NOT_DUMPED == issue.check) {
          bool expected = false;

          if (!m_users_not_dumped_shown.compare_exchange_strong(expected,
                                                                true)) {
            show_issue = false;
          }
        }

        if (show_issue) {
          console->print_warning(issue.description, issue_attribs(issue));
        }
        break;
      }

      case Compatibility_issue::Status::ERROR: {
        status.set(issues::Status::ERROR);

        std::string hint;

        if (issue.compatibility_options.empty()) {
          hint = "this issue needs to be fixed manually";
        }

        if (Compatibility_check::TABLE_MISSING_PK == issue.check) {
          status.set(issues::Status::ERROR_MISSING_PKS);
        } else if (Compatibility_check::USER_WILDCARD_GRANT == issue.check) {
          status.set(issues::Status::ERROR_HAS_WILDCARD_GRANTS);
        } else if (Compatibility_check::VIEW_MISMATCHED_REFERENCE ==
                   issue.check) {
          status.set(issues::Status::WARNING_HAS_MISMATCHED_VIEW_REFERENCES);
        } else {
          if (Compatibility_check::USER_INVALID_GRANTS == issue.check) {
            status.set(issues::Status::ERROR_HAS_INVALID_GRANTS);
          }

          if (!issue.compatibility_options.empty()) {
            std::vector<std::string> options;

            for (auto option : issue.compatibility_options.values()) {
              options.emplace_back("'" + to_string(option) + "'");
            }

            if (1 == options.size()) {
              hint = std::move(options[0]);
            } else {
              hint = "either ";
              hint += options.front();

              if (options.size() > 2) {
                const auto end = std::prev(options.end());

                for (auto it = std::next(options.begin()); it != end; ++it) {
                  hint += ", ";
                  hint += *it;
                }
              }

              hint += " or ";
              hint += options.back();
            }

            hint = "fix this with " + hint + " compatibility option";
          }
        }

        console->print_error(
            issue.description + (hint.empty() ? "" : " (" + hint + ")"),
            issue_attribs(issue));
        break;
      }
    }
  }

  return status;
}

bool Dumper::should_fetch_data_masking_policies() const noexcept {
  // // WL17279-FR1.5: if we're not dumping DDL then we're not dumping policies
  return m_data_masking_enabled && m_options.dump_ddl() &&
         m_options.dump_data_masking_policies();
}

bool Dumper::should_dump_data_masking_policies() const noexcept {
  // cache needs to be already initialized
  assert(m_cache_initialized);

  return should_fetch_data_masking_policies() &&
         !m_cache.data_masking_policies.empty();
}

bool Dumper::will_dump_masked_table_ddl() const noexcept {
  // cache needs to be already initialized
  assert(m_cache_initialized);

  return m_data_masking_enabled && m_options.dump_ddl() &&
         m_cache.has_masking_policy_tables;
}

bool Dumper::will_dump_table_data_with_masked_columns() const noexcept {
  // cache needs to be already initialized
  assert(m_cache_initialized);

  return m_data_masking_enabled && m_options.dump_data() &&
         m_cache.has_masking_policy_tables;
}

}  // namespace dump
}  // namespace mysqlsh
