From 5b5bb4534bd764ec7dbb6825851bf5f41adc664c Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 28 Nov 2022 01:07:16 -0800 Subject: [PATCH 01/12] Refactoring for Custom generator --- .../target/include/target/custom_generator.h | 181 +++++++++--------- .../custom_generator_context.h | 2 +- .../src/custom_generator/custom_generator.cpp | 47 +++-- 3 files changed, 115 insertions(+), 115 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 0712cde5..2fc0c6f8 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -51,7 +51,6 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { } }; - using UserIdPair = std::pair; std::unordered_map ids; void ConvertToInternal() { @@ -64,6 +63,92 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { } }; +struct Comparator { + Comparator(const buildcc::internal::CustomGeneratorSchema &loaded, + const buildcc::UserCustomGeneratorSchema ¤t) + : loaded_schema_(loaded), current_schema_(current) {} + + enum class State { + kRemoved, + kAdded, + kCheckLater, + }; + + void AddAllIds() { + const auto &curr_ids = current_schema_.ids; + for (const auto &[id, _] : curr_ids) { + id_state_info_.at(State::kAdded).insert(id); + } + } + + void CompareIds() { + const auto &prev_ids = loaded_schema_.internal_ids; + const auto &curr_ids = current_schema_.ids; + + for (const auto &[prev_id, _] : prev_ids) { + if (curr_ids.find(prev_id) == curr_ids.end()) { + // Id Removed condition, previous id is not present in the current run + id_state_info_.at(State::kRemoved).insert(prev_id); + } + } + + for (const auto &[curr_id, _] : curr_ids) { + if (prev_ids.find(curr_id) == prev_ids.end()) { + // Id Added condition + id_state_info_.at(State::kAdded).insert(curr_id); + } else { + // Id Check Later condition + id_state_info_.at(State::kCheckLater).insert(curr_id); + } + } + } + + bool IsChanged(const std::string &id) const { + const auto &previous_id_info = loaded_schema_.internal_ids.at(id); + const auto ¤t_id_info = current_schema_.ids.at(id); + + bool changed = + buildcc::internal::CheckPaths(previous_id_info.internal_inputs, + current_id_info.internal_inputs) != + buildcc::internal::PathState::kNoChange; + changed = changed || buildcc::internal::CheckChanged( + previous_id_info.outputs, current_id_info.outputs); + if (!changed && current_id_info.blob_handler != nullptr) { + // We only check blob handler if not changed by inputs/outputs + // Checking blob_handler could be expensive so this optimization is made + // to run only when changed == false + changed = current_id_info.blob_handler->CheckChanged( + previous_id_info.userblob, current_id_info.userblob); + } + return changed; + } + + const std::unordered_set &GetRemovedIds() const { + return id_state_info_.at(State::kRemoved); + } + + const std::unordered_set &GetAddedIds() const { + return id_state_info_.at(State::kAdded); + } + + const std::unordered_set &GetCheckLaterIds() const { + return id_state_info_.at(State::kCheckLater); + } + + bool IsIdAdded(const std::string &id) const { + return id_state_info_.at(State::kAdded).count(id) == 1; + } + +private: + const buildcc::internal::CustomGeneratorSchema &loaded_schema_; + const buildcc::UserCustomGeneratorSchema ¤t_schema_; + std::unordered_map> id_state_info_{ + {State::kRemoved, std::unordered_set()}, + {State::kAdded, std::unordered_set()}, + {State::kCheckLater, std::unordered_set()}, + }; +}; + class CustomGenerator : public internal::BuilderInterface { public: CustomGenerator(const std::string &name, const TargetEnv &env) @@ -115,92 +200,6 @@ class CustomGenerator : public internal::BuilderInterface { const fs::path &GetBuildDir() const { return env_.GetTargetBuildDir(); } const std::string &Get(const std::string &file_identifier) const; -private: - struct Comparator { - Comparator(const internal::CustomGeneratorSchema &loaded, - const UserCustomGeneratorSchema &us) - : loaded_schema_(loaded), current_schema_(us) {} - - enum class State { - kRemoved, - kAdded, - kCheckLater, - }; - - void AddAllIds() { - const auto &curr_ids = current_schema_.ids; - for (const auto &[id, _] : curr_ids) { - id_state_info_.at(State::kAdded).insert(id); - } - } - - void CompareIds() { - const auto &prev_ids = loaded_schema_.internal_ids; - const auto &curr_ids = current_schema_.ids; - - for (const auto &[prev_id, _] : prev_ids) { - if (curr_ids.find(prev_id) == curr_ids.end()) { - // Id Removed condition, previous id is not present in the current run - id_state_info_.at(State::kRemoved).insert(prev_id); - } - } - - for (const auto &[curr_id, _] : curr_ids) { - if (prev_ids.find(curr_id) == prev_ids.end()) { - // Id Added condition - id_state_info_.at(State::kAdded).insert(curr_id); - } else { - // Id Check Later condition - id_state_info_.at(State::kCheckLater).insert(curr_id); - } - } - } - - bool IsChanged(const std::string &id) const { - const auto &previous_id_info = loaded_schema_.internal_ids.at(id); - const auto ¤t_id_info = current_schema_.ids.at(id); - - bool changed = internal::CheckPaths(previous_id_info.internal_inputs, - current_id_info.internal_inputs) != - internal::PathState::kNoChange; - changed = changed || internal::CheckChanged(previous_id_info.outputs, - current_id_info.outputs); - if (!changed && current_id_info.blob_handler != nullptr) { - // We only check blob handler if not changed by inputs/outputs - // Checking blob_handler could be expensive so this optimization is made - // to run only when changed == false - changed = current_id_info.blob_handler->CheckChanged( - previous_id_info.userblob, current_id_info.userblob); - } - return changed; - } - - const std::unordered_set &GetRemovedIds() const { - return id_state_info_.at(State::kRemoved); - } - - const std::unordered_set &GetAddedIds() const { - return id_state_info_.at(State::kAdded); - } - - const std::unordered_set &GetCheckLaterIds() const { - return id_state_info_.at(State::kCheckLater); - } - - bool IsIdAdded(const std::string &id) const { - return id_state_info_.at(State::kAdded).count(id) == 1; - } - - private: - const internal::CustomGeneratorSchema &loaded_schema_; - const UserCustomGeneratorSchema ¤t_schema_; - std::unordered_map> id_state_info_{ - {State::kRemoved, std::unordered_set()}, - {State::kAdded, std::unordered_set()}, - {State::kCheckLater, std::unordered_set()}, - }; - }; - private: void Initialize(); @@ -227,15 +226,17 @@ class CustomGenerator : public internal::BuilderInterface { // Serialization UserCustomGeneratorSchema user_; + // Internal + env::Command command_; + // Comparator + // TODO, Shift internally Comparator comparator_; + // TODO, Shift internally std::mutex success_schema_mutex_; std::unordered_map success_schema_; - - // Internal - env::Command command_; }; } // namespace buildcc diff --git a/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h b/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h index 207015d9..5619eae1 100644 --- a/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h +++ b/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h @@ -37,7 +37,7 @@ class CustomGeneratorContext { }; // clang-format off -using GenerateCb = std::function; +using GenerateCb = std::function; // clang-format on } // namespace buildcc diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index b60283e8..3d8e2ad2 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -76,7 +76,6 @@ void CustomGenerator::AddIdInfo( void CustomGenerator::Build() { (void)serialization_.LoadFromFile(); - GenerateTask(); } @@ -101,29 +100,6 @@ void CustomGenerator::Initialize() { tf_.name(name_); } -void CustomGenerator::BuildGenerate() { - if (!serialization_.IsLoaded()) { - comparator_.AddAllIds(); - dirty_ = true; - } else { - // For IDS - comparator_.CompareIds(); - - const bool is_removed = !comparator_.GetRemovedIds().empty(); - const bool is_added = !comparator_.GetAddedIds().empty(); - dirty_ = is_removed || is_added; - - if (is_removed) { - IdRemoved(); - } - - for (const auto &id : comparator_.GetAddedIds()) { - (void)id; - IdAdded(); - } - } -} - void CustomGenerator::GenerateTask() { tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { if (env::get_task_state() != env::TaskState::SUCCESS) { @@ -167,6 +143,29 @@ void CustomGenerator::GenerateTask() { generate_task.name(kGenerateTaskName); } +void CustomGenerator::BuildGenerate() { + if (!serialization_.IsLoaded()) { + comparator_.AddAllIds(); + dirty_ = true; + } else { + // For IDS + comparator_.CompareIds(); + + const bool is_removed = !comparator_.GetRemovedIds().empty(); + const bool is_added = !comparator_.GetAddedIds().empty(); + dirty_ = is_removed || is_added; + + if (is_removed) { + IdRemoved(); + } + + for (const auto &id : comparator_.GetAddedIds()) { + (void)id; + IdAdded(); + } + } +} + tf::Task CustomGenerator::CreateTaskRunner(tf::Subflow &subflow, const std::string &id) { return subflow.emplace([&, id]() { From 106025b387e773ebb951dec789e99364756fba06 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 28 Nov 2022 03:15:48 -0800 Subject: [PATCH 02/12] Added PathInfoList and PathList APIs Migrated CustomGenerator with PathInfoList and PathList --- .../target/include/target/custom_generator.h | 17 +- .../custom_generator_context.h | 9 +- .../src/custom_generator/custom_generator.cpp | 13 +- .../test/target/test_custom_generator.cpp | 42 ++--- .../include/schema/custom_generator_schema.h | 8 +- buildcc/schema/include/schema/path.h | 154 ++++++++++++++++++ 6 files changed, 197 insertions(+), 46 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 2fc0c6f8..03120d4f 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -39,13 +39,13 @@ namespace buildcc { struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { struct UserIdInfo : internal::CustomGeneratorSchema::IdInfo { - fs_unordered_set inputs; // TODO, Remove GenerateCb generate_cb; std::shared_ptr blob_handler{nullptr}; void ConvertToInternal() { - internal_inputs = internal::path_schema_convert( - inputs, internal::Path::CreateExistingPath); + for (const auto &[path_str, _] : inputs.GetPathInfos()) { + inputs.ComputeHash(path_str); + } userblob = blob_handler != nullptr ? blob_handler->GetSerializedData() : std::vector(); } @@ -55,8 +55,7 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { void ConvertToInternal() { for (auto &[id_key, id_info] : ids) { - id_info.internal_inputs = path_schema_convert( - id_info.inputs, internal::Path::CreateExistingPath); + id_info.ConvertToInternal(); auto [_, success] = internal_ids.try_emplace(id_key, id_info); env::assert_fatal(success, fmt::format("Could not save {}", id_key)); } @@ -107,12 +106,8 @@ struct Comparator { const auto &previous_id_info = loaded_schema_.internal_ids.at(id); const auto ¤t_id_info = current_schema_.ids.at(id); - bool changed = - buildcc::internal::CheckPaths(previous_id_info.internal_inputs, - current_id_info.internal_inputs) != - buildcc::internal::PathState::kNoChange; - changed = changed || buildcc::internal::CheckChanged( - previous_id_info.outputs, current_id_info.outputs); + bool changed = !previous_id_info.inputs.IsEqual(current_id_info.inputs) || + !previous_id_info.outputs.IsEqual(current_id_info.outputs); if (!changed && current_id_info.blob_handler != nullptr) { // We only check blob handler if not changed by inputs/outputs // Checking blob_handler could be expensive so this optimization is made diff --git a/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h b/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h index 5619eae1..4c49d5ff 100644 --- a/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h +++ b/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h @@ -25,14 +25,15 @@ namespace buildcc { class CustomGeneratorContext { public: - CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, - const fs_unordered_set &o, + CustomGeneratorContext(const env::Command &c, + const std::unordered_set &i, + const std::unordered_set &o, const std::vector &ub) : command(c), inputs(i), outputs(o), userblob(ub) {} const env::Command &command; - const fs_unordered_set &inputs; - const fs_unordered_set &outputs; + const std::unordered_set &inputs; + const std::unordered_set &outputs; const std::vector &userblob; }; diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 3d8e2ad2..1bae5c1a 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -62,12 +62,12 @@ void CustomGenerator::AddIdInfo( UserCustomGeneratorSchema::UserIdInfo schema; for (const auto &i : inputs) { - fs::path input = string_as_path(command_.Construct(i)); - schema.inputs.emplace(std::move(input)); + auto input = command_.Construct(i); + schema.inputs.Emplace(input, 0); } for (const auto &o : outputs) { - fs::path output = string_as_path(command_.Construct(o)); - schema.outputs.emplace(std::move(output)); + auto output = command_.Construct(o); + schema.outputs.Emplace(output); } schema.generate_cb = generate_cb; schema.blob_handler = blob_handler; @@ -191,8 +191,9 @@ void CustomGenerator::TaskRunner(const std::string &id) { const auto ¤t_id_info = user_.ids.at(id); if (run) { dirty_ = true; - CustomGeneratorContext ctx(command_, current_id_info.inputs, - current_id_info.outputs, + const auto input_paths = current_id_info.inputs.GetPaths(); + const auto &output_paths = current_id_info.outputs.GetPaths(); + CustomGeneratorContext ctx(command_, input_paths, output_paths, current_id_info.userblob); bool success = current_id_info.generate_cb(ctx); env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id)); diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 839c91f7..39818051 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -54,12 +54,12 @@ TEST(CustomGeneratorTestGroup, Basic) { const auto &internal_map = serialization.GetLoad().internal_ids; CHECK_EQUAL(internal_map.size(), 2); const auto &id1_info = internal_map.at("id1"); - CHECK_EQUAL(id1_info.internal_inputs.size(), 1); - CHECK_EQUAL(id1_info.outputs.size(), 1); + CHECK_EQUAL(id1_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id1_info.outputs.GetPaths().size(), 1); const auto &id2_info = internal_map.at("id2"); - CHECK_EQUAL(id2_info.internal_inputs.size(), 1); - CHECK_EQUAL(id2_info.outputs.size(), 0); + CHECK_EQUAL(id2_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id2_info.outputs.GetPaths().size(), 0); } } @@ -184,12 +184,12 @@ TEST(CustomGeneratorTestGroup, DefaultArgumentUsage) { const auto &internal_map = serialization.GetLoad().internal_ids; CHECK_EQUAL(internal_map.size(), 2); const auto &id1_info = internal_map.at("id1"); - CHECK_EQUAL(id1_info.internal_inputs.size(), 1); - CHECK_EQUAL(id1_info.outputs.size(), 1); + CHECK_EQUAL(id1_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id1_info.outputs.GetPaths().size(), 1); const auto &id2_info = internal_map.at("id2"); - CHECK_EQUAL(id2_info.internal_inputs.size(), 1); - CHECK_EQUAL(id2_info.outputs.size(), 0); + CHECK_EQUAL(id2_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id2_info.outputs.GetPaths().size(), 0); } } @@ -332,11 +332,11 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK_TRUE(serialization.LoadFromFile()); CHECK_EQUAL(serialization.GetLoad().internal_ids.size(), 2); auto imap = serialization.GetLoad().internal_ids; - CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id1").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id2").inputs.GetPathInfos().size(), 1); - CHECK_EQUAL(imap.at("id1").outputs.size(), 1); - CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.GetPaths().size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.GetPaths().size(), 1); } buildcc::m::blocking_sleep(1); @@ -368,14 +368,14 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK_TRUE(serialization.LoadFromFile()); CHECK_EQUAL(serialization.GetLoad().internal_ids.size(), 2); auto imap = serialization.GetLoad().internal_ids; - CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id1").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.GetPaths().size(), 1); - CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.GetPaths().size(), 1); CHECK_EQUAL(last_write_timestamp, - imap.at("id2").internal_inputs.begin()->last_write_timestamp); + imap.at("id2").inputs.GetPathInfos().begin()->second); CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); } @@ -400,11 +400,11 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK_TRUE(serialization.LoadFromFile()); CHECK_EQUAL(serialization.GetLoad().internal_ids.size(), 2); auto imap = serialization.GetLoad().internal_ids; - CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id1").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.GetPaths().size(), 1); - CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.GetPaths().size(), 1); CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); } diff --git a/buildcc/schema/include/schema/custom_generator_schema.h b/buildcc/schema/include/schema/custom_generator_schema.h index d7027b98..a4088a66 100644 --- a/buildcc/schema/include/schema/custom_generator_schema.h +++ b/buildcc/schema/include/schema/custom_generator_schema.h @@ -37,18 +37,18 @@ struct CustomGeneratorSchema { static constexpr const char *const kUserblob = "userblob"; public: - path_unordered_set internal_inputs; - fs_unordered_set outputs; + PathInfoList inputs; + PathList outputs; std::vector userblob; friend void to_json(json &j, const IdInfo &info) { - j[kInputs] = info.internal_inputs; + j[kInputs] = info.inputs; j[kOutputs] = info.outputs; j[kUserblob] = info.userblob; } friend void from_json(const json &j, IdInfo &info) { - j.at(kInputs).get_to(info.internal_inputs); + j.at(kInputs).get_to(info.inputs); j.at(kOutputs).get_to(info.outputs); j.at(kUserblob).get_to(info.userblob); } diff --git a/buildcc/schema/include/schema/path.h b/buildcc/schema/include/schema/path.h index 755fa6db..80586a90 100644 --- a/buildcc/schema/include/schema/path.h +++ b/buildcc/schema/include/schema/path.h @@ -92,6 +92,43 @@ struct Path { return pathname == other_pathname; } + /** + * @brief Sanitizes a fs::path or std::string to a standard path string + * - Converts backslash (\) to forward slash (/) + * - Makes fs::lexically_normal (see std::filesystem library impl) + * + * @param str User provided fs::path/std::string + * @return std::string Sanitized path as std::string + */ + static std::string ToPathString(const std::string &str) { + auto path_str = str; + std::replace(path_str.begin(), path_str.end(), '\\', '/'); + path_str = fs::path(path_str).lexically_normal().string(); + return path_str; + } + + /** + * @brief Formats a fs::path or std::string for display + * - All the sanitization as done in `ToPathString` + * Additionally + * - Adds quotation marks ("") when a space is detected + * For example: test/hello world/ -> "test/hello world/" + * + * NOTE: Use this API only in places where you would like to output to + * console/run or create command through subprocess + * + * @param str User provided fs::path/std::string + * @return std::string Sanitized path as std::string for display + */ + static std::string ToPathDisplayString(const std::string &str) { + auto path_str = ToPathString(str); + // if spaces are present in the path string, surround this with brackets + if (path_str.find(' ') != std::string::npos) { + path_str = fmt::format("\"{}\"", path_str); + } + return path_str; + } + // JSON specialization friend void to_json(json &j, const Path &p) { @@ -135,6 +172,118 @@ class PathHash { using path_unordered_set = std::unordered_set; using fs_unordered_set = std::unordered_set; +/** + * @brief Stores path + path hash in a hashmap + * + */ +class PathInfoList { +private: + static constexpr const char *const kPath = "path"; + static constexpr const char *const kHash = "hash"; + +public: + PathInfoList() = default; + explicit PathInfoList( + std::initializer_list> + path_infos) { + for (const auto &pinfo : path_infos) { + Emplace(pinfo.first, pinfo.second); + } + } + + void Emplace(const std::string &pstr, std::uint64_t hash) { + auto path_str = Path::ToPathString(pstr); + path_infos_.emplace(std::move(path_str), std::move(hash)); + } + + // TODO, Add Compute Strategy here + void ComputeHash(const std::string &pstr) { + auto path_str = Path::ToPathString(pstr); + const bool found = path_infos_.find(path_str) != path_infos_.end(); + // TODO, Verbose error message + env::assert_fatal(found, ""); + + std::error_code errcode; + const std::uint64_t last_write_timestamp = + std::filesystem::last_write_time(path_str, errcode) + .time_since_epoch() + .count(); + env::assert_fatal(errcode.value() == 0, + fmt::format("{} not found", path_str)); + path_infos_.at(path_str) = last_write_timestamp; + } + + // TODO, Use this later + void ComputeHashForAll() {} + + // TODO, Change return value here to std::string + std::uint64_t GetHash(const std::string &str) const { + auto path_str = Path::ToPathString(str); + const bool found = path_infos_.find(path_str) != path_infos_.end(); + env::assert_fatal(found, ""); + return path_infos_.at(path_str); + } + + bool IsEqual(const PathInfoList &other) const { + return path_infos_ == other.path_infos_; + } + + const std::unordered_map &GetPathInfos() const { + return path_infos_; + } + + std::unordered_set GetPaths() const { + std::unordered_set paths; + for (const auto &[path_str, hash] : path_infos_) { + paths.emplace(path_str); + } + return paths; + } + + friend void to_json(json &j, const PathInfoList &plist) { + j = plist.path_infos_; + } + + friend void from_json(const json &j, PathInfoList &plist) { + j.get_to(plist.path_infos_); + } + +private: + std::unordered_map path_infos_; +}; + +/** + * @brief Stores path + */ +class PathList { +public: + PathList() = default; + PathList(std::initializer_list paths) { + for (const auto &path : paths) { + Emplace(path); + } + } + + void Emplace(const std::string &pstr) { + auto path_str = Path::ToPathString(pstr); + paths_.emplace(std::move(path_str)); + } + + bool IsEqual(const PathList &other) const { return paths_ == other.paths_; } + const std::unordered_set &GetPaths() const { return paths_; } + + // TODO, Add a Contains API + + friend void to_json(json &j, const PathList &plist) { j = plist.paths_; } + + friend void from_json(const json &j, PathList &plist) { + j.get_to(plist.paths_); + } + +private: + std::unordered_set paths_; +}; + inline std::vector path_schema_convert(const std::vector &path_list, const std::function &cb = @@ -170,6 +319,11 @@ path_schema_convert(const path_unordered_set &internal_path_set) { namespace buildcc { +inline std::string sanitize_path_string(const fs::path &p) { + auto path_str = p.lexically_normal().string(); + return path_str; +} + inline std::string path_as_string(const fs::path &p) { return internal::Path(p).GetPathAsString(); } From 7646cdffda4879185a094923ff9cf754bbd4f992 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 28 Nov 2022 03:43:20 -0800 Subject: [PATCH 03/12] Converted hash from uint64_t to std::string --- .../target/include/target/custom_generator.h | 4 +- .../src/custom_generator/custom_generator.cpp | 2 +- .../test/target/test_custom_generator.cpp | 6 +- buildcc/schema/include/schema/path.h | 124 +++++++++--------- 4 files changed, 67 insertions(+), 69 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 03120d4f..1c976e2f 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -43,9 +43,7 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { std::shared_ptr blob_handler{nullptr}; void ConvertToInternal() { - for (const auto &[path_str, _] : inputs.GetPathInfos()) { - inputs.ComputeHash(path_str); - } + inputs.ComputeHashForAll(); userblob = blob_handler != nullptr ? blob_handler->GetSerializedData() : std::vector(); } diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 1bae5c1a..3ce2ab87 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -63,7 +63,7 @@ void CustomGenerator::AddIdInfo( UserCustomGeneratorSchema::UserIdInfo schema; for (const auto &i : inputs) { auto input = command_.Construct(i); - schema.inputs.Emplace(input, 0); + schema.inputs.Emplace(input, ""); } for (const auto &o : outputs) { auto output = command_.Construct(o); diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 39818051..f4293c9f 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -348,7 +348,7 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { buildcc::env::save_file( (cgen.GetBuildDir() / "dummy_main.cpp").string().c_str(), "", false); - std::uint64_t last_write_timestamp = static_cast( + auto last_write_timestamp = static_cast( fs::last_write_time(cgen.GetBuildDir() / "dummy_main.cpp") .time_since_epoch() .count()); @@ -374,8 +374,8 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK_EQUAL(imap.at("id2").inputs.GetPathInfos().size(), 1); CHECK_EQUAL(imap.at("id2").outputs.GetPaths().size(), 1); - CHECK_EQUAL(last_write_timestamp, - imap.at("id2").inputs.GetPathInfos().begin()->second); + STRCMP_EQUAL(std::to_string(last_write_timestamp).c_str(), + imap.at("id2").inputs.GetPathInfos().begin()->second.c_str()); CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); } diff --git a/buildcc/schema/include/schema/path.h b/buildcc/schema/include/schema/path.h index 80586a90..9e1107d3 100644 --- a/buildcc/schema/include/schema/path.h +++ b/buildcc/schema/include/schema/path.h @@ -34,6 +34,7 @@ using json = nlohmann::ordered_json; namespace buildcc::internal { +// TODO, Update this struct Path { private: static constexpr const char *const kPathName = "path"; @@ -161,6 +162,7 @@ struct Path { std::uint64_t last_write_timestamp{0}; }; +// TODO, Remove this // Used by Path class PathHash { public: @@ -169,9 +171,40 @@ class PathHash { size_t operator()(const fs::path &p) const { return fs::hash_value(p); } }; +// TODO, Remove this using path_unordered_set = std::unordered_set; using fs_unordered_set = std::unordered_set; +/** + * @brief Stores path + */ +class PathList { +public: + PathList() = default; + PathList(std::initializer_list paths) { + for (const auto &path : paths) { + Emplace(path); + } + } + + void Emplace(const std::string &pstr) { + auto path_str = Path::ToPathString(pstr); + paths_.emplace(std::move(path_str)); + } + + bool IsEqual(const PathList &other) const { return paths_ == other.paths_; } + const std::unordered_set &GetPaths() const { return paths_; } + + friend void to_json(json &j, const PathList &plist) { j = plist.paths_; } + + friend void from_json(const json &j, PathList &plist) { + j.get_to(plist.paths_); + } + +private: + std::unordered_set paths_; +}; + /** * @brief Stores path + path hash in a hashmap * @@ -184,40 +217,25 @@ class PathInfoList { public: PathInfoList() = default; explicit PathInfoList( - std::initializer_list> + std::initializer_list> path_infos) { for (const auto &pinfo : path_infos) { Emplace(pinfo.first, pinfo.second); } } - void Emplace(const std::string &pstr, std::uint64_t hash) { + void Emplace(const std::string &pstr, const std::string &hash) { auto path_str = Path::ToPathString(pstr); - path_infos_.emplace(std::move(path_str), std::move(hash)); + path_infos_.emplace(std::move(path_str), hash); } - // TODO, Add Compute Strategy here - void ComputeHash(const std::string &pstr) { - auto path_str = Path::ToPathString(pstr); - const bool found = path_infos_.find(path_str) != path_infos_.end(); - // TODO, Verbose error message - env::assert_fatal(found, ""); - - std::error_code errcode; - const std::uint64_t last_write_timestamp = - std::filesystem::last_write_time(path_str, errcode) - .time_since_epoch() - .count(); - env::assert_fatal(errcode.value() == 0, - fmt::format("{} not found", path_str)); - path_infos_.at(path_str) = last_write_timestamp; + void ComputeHashForAll() { + for (auto &[path_str, hash] : path_infos_) { + hash = ComputeHash(path_str); + } } - // TODO, Use this later - void ComputeHashForAll() {} - - // TODO, Change return value here to std::string - std::uint64_t GetHash(const std::string &str) const { + const std::string &GetHash(const std::string &str) const { auto path_str = Path::ToPathString(str); const bool found = path_infos_.find(path_str) != path_infos_.end(); env::assert_fatal(found, ""); @@ -228,7 +246,7 @@ class PathInfoList { return path_infos_ == other.path_infos_; } - const std::unordered_map &GetPathInfos() const { + const std::unordered_map &GetPathInfos() const { return path_infos_; } @@ -240,6 +258,22 @@ class PathInfoList { return paths; } + // TODO, Add Compute Strategy enum + static std::string ComputeHash(const std::string &pstr) { + auto path_str = Path::ToPathString(pstr); + + // TODO, There might be a file checksum hash compute strategy + // This is the timestamp hash compute strategy + std::error_code errcode; + const std::uint64_t last_write_timestamp = + std::filesystem::last_write_time(path_str, errcode) + .time_since_epoch() + .count(); + env::assert_fatal(errcode.value() == 0, + fmt::format("{} not found", path_str)); + return std::to_string(last_write_timestamp); + } + friend void to_json(json &j, const PathInfoList &plist) { j = plist.path_infos_; } @@ -249,41 +283,10 @@ class PathInfoList { } private: - std::unordered_map path_infos_; -}; - -/** - * @brief Stores path - */ -class PathList { -public: - PathList() = default; - PathList(std::initializer_list paths) { - for (const auto &path : paths) { - Emplace(path); - } - } - - void Emplace(const std::string &pstr) { - auto path_str = Path::ToPathString(pstr); - paths_.emplace(std::move(path_str)); - } - - bool IsEqual(const PathList &other) const { return paths_ == other.paths_; } - const std::unordered_set &GetPaths() const { return paths_; } - - // TODO, Add a Contains API - - friend void to_json(json &j, const PathList &plist) { j = plist.paths_; } - - friend void from_json(const json &j, PathList &plist) { - j.get_to(plist.paths_); - } - -private: - std::unordered_set paths_; + std::unordered_map path_infos_; }; +// TODO, Remove this inline std::vector path_schema_convert(const std::vector &path_list, const std::function &cb = @@ -295,6 +298,7 @@ path_schema_convert(const std::vector &path_list, return internal_path_list; } +// TODO, Remove this inline path_unordered_set path_schema_convert(const fs_unordered_set &path_set, const std::function &cb = @@ -306,6 +310,7 @@ path_schema_convert(const fs_unordered_set &path_set, return internal_path_set; } +// TODO, Remove this inline fs_unordered_set path_schema_convert(const path_unordered_set &internal_path_set) { fs_unordered_set path_set; @@ -319,11 +324,6 @@ path_schema_convert(const path_unordered_set &internal_path_set) { namespace buildcc { -inline std::string sanitize_path_string(const fs::path &p) { - auto path_str = p.lexically_normal().string(); - return path_str; -} - inline std::string path_as_string(const fs::path &p) { return internal::Path(p).GetPathAsString(); } From 216cf2efe32bb0d870ca88be4ea092181fbbbb31 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 28 Nov 2022 04:04:04 -0800 Subject: [PATCH 04/12] Added TypedCustomBlobHandler for custom generator --- .../custom_generator/custom_blob_handler.h | 48 +++++++++++++++++++ .../target/src/generator/file_generator.cpp | 41 +++------------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h b/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h index 1c268301..3b64d23c 100644 --- a/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h +++ b/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h @@ -23,6 +23,12 @@ namespace buildcc { +/** + * @brief Abstract class for serializing additional data for which rebuilds + * might be triggered i.e data that is not input/output files + * TODO, Add examples here + * + */ class CustomBlobHandler { public: CustomBlobHandler() = default; @@ -54,6 +60,48 @@ class CustomBlobHandler { virtual std::vector Serialize() const = 0; }; +/** + * @brief Typed Custom Blob handler which automatically performs Serialization + * and Deserialization as long as it is JSON serializable + * + * NOTE: Type data is stored as a reference (to avoid copying large amount of + * data) when constructing TypedCustomBlobHandler + * + * @tparam Type should be JSON serializable (see nlohmann::json compatible + * objects) + */ +template +class TypedCustomBlobHandler : public CustomBlobHandler { +public: + explicit TypedCustomBlobHandler(const Type &data) : data_(data) {} + + // serialized_data has already been verified + static Type Deserialize(const std::vector &serialized_data) { + json j = json::from_msgpack(serialized_data, true, false); + Type deserialized; + j.get_to(deserialized); + return deserialized; + } + +private: + const Type &data_; + + bool Verify(const std::vector &serialized_data) const override { + json j = json::from_msgpack(serialized_data, true, false); + return !j.is_discarded(); + } + + bool IsEqual(const std::vector &previous, + const std::vector ¤t) const override { + return Deserialize(previous) == Deserialize(current); + } + + std::vector Serialize() const override { + json j = data_; + return json::to_msgpack(j); + } +}; + } // namespace buildcc #endif diff --git a/buildcc/lib/target/src/generator/file_generator.cpp b/buildcc/lib/target/src/generator/file_generator.cpp index 5cdb592f..3256ad2b 100644 --- a/buildcc/lib/target/src/generator/file_generator.cpp +++ b/buildcc/lib/target/src/generator/file_generator.cpp @@ -22,44 +22,12 @@ namespace { -class FileGeneratorBlobHandler : public buildcc::CustomBlobHandler { -public: - explicit FileGeneratorBlobHandler(const std::vector &commands) - : commands_(commands) {} - - // serialized_data has already been verified - static std::vector - Deserialize(const std::vector &serialized_data) { - json j = json::from_msgpack(serialized_data, true, false); - std::vector deserialized; - j.get_to(deserialized); - return deserialized; - } - -private: - const std::vector &commands_; - - bool Verify(const std::vector &serialized_data) const override { - json j = json::from_msgpack(serialized_data, true, false); - return !j.is_discarded(); - } - - bool IsEqual(const std::vector &previous, - const std::vector ¤t) const override { - return Deserialize(previous) == Deserialize(current); - } - - std::vector Serialize() const override { - json j = commands_; - return json::to_msgpack(j); - } -}; - bool FileGeneratorGenerateCb(const buildcc::CustomGeneratorContext &ctx) { (void)ctx; bool success = true; std::vector commands = - FileGeneratorBlobHandler::Deserialize(ctx.userblob); + buildcc::TypedCustomBlobHandler>::Deserialize( + ctx.userblob); for (const auto &c : commands) { bool executed = buildcc::env::Command::Execute(c); if (!executed) { @@ -92,8 +60,11 @@ void FileGenerator::AddCommand( } void FileGenerator::Build() { + auto file_blob_handler = + std::make_shared>>( + commands_); AddIdInfo("Generate", inputs_, outputs_, FileGeneratorGenerateCb, - std::make_shared(commands_)); + file_blob_handler); this->CustomGenerator::Build(); } From 0322a0bd957f12c817ad1ee9c89933cc642fad0a Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 28 Nov 2022 06:13:28 -0800 Subject: [PATCH 05/12] Minor updates to custom_generator --- buildcc/lib/target/include/target/custom_generator.h | 10 +++++----- .../target/src/custom_generator/custom_generator.cpp | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 1c976e2f..774ac455 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -39,17 +39,15 @@ namespace buildcc { struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { struct UserIdInfo : internal::CustomGeneratorSchema::IdInfo { - GenerateCb generate_cb; - std::shared_ptr blob_handler{nullptr}; - void ConvertToInternal() { inputs.ComputeHashForAll(); userblob = blob_handler != nullptr ? blob_handler->GetSerializedData() : std::vector(); } - }; - std::unordered_map ids; + GenerateCb generate_cb; + std::shared_ptr blob_handler{nullptr}; + }; void ConvertToInternal() { for (auto &[id_key, id_info] : ids) { @@ -58,6 +56,8 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { env::assert_fatal(success, fmt::format("Could not save {}", id_key)); } } + + std::unordered_map ids; }; struct Comparator { diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 3ce2ab87..63dfbd80 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -129,8 +129,8 @@ void CustomGenerator::GenerateTask() { UserCustomGeneratorSchema user_final_schema; user_final_schema.ids.insert(success_schema_.begin(), success_schema_.end()); - user_final_schema.ConvertToInternal(); + serialization_.UpdateStore(user_final_schema); env::assert_fatal(serialization_.StoreToFile(), fmt::format("Store failed for {}", name_)); @@ -139,7 +139,6 @@ void CustomGenerator::GenerateTask() { env::set_task_state(env::TaskState::FAILURE); } }); - // TODO, Instead of "Generate" name the task of user's choice generate_task.name(kGenerateTaskName); } From 7ae65ea322255caa6fcab01ecabd41493e1c4cc4 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 28 Nov 2022 23:47:21 -0800 Subject: [PATCH 06/12] Updated custom generator --- .../target/include/target/custom_generator.h | 6 ++- .../src/custom_generator/custom_generator.cpp | 42 ++++++++++--------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 774ac455..f7ef0d02 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -196,8 +196,10 @@ class CustomGenerator : public internal::BuilderInterface { private: void Initialize(); - tf::Task CreateTaskRunner(tf::Subflow &subflow, const std::string &id); - void TaskRunner(const std::string &id); + tf::Task CreateTaskRunner(tf::Subflow &subflow, const std::string &id, + UserCustomGeneratorSchema::UserIdInfo &id_info); + void TaskRunner(const std::string &id, + const UserCustomGeneratorSchema::UserIdInfo &id_info); void GenerateTask(); void BuildGenerate(); diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 63dfbd80..ee396aae 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -112,13 +112,13 @@ void CustomGenerator::GenerateTask() { // Create runner for each added/updated id for (const auto &id : comparator_.GetAddedIds()) { - auto task = CreateTaskRunner(subflow, id); - task.name(id); + auto &id_info = user_.ids.at(id); + CreateTaskRunner(subflow, id, id_info); } for (const auto &id : comparator_.GetCheckLaterIds()) { - auto task = CreateTaskRunner(subflow, id); - task.name(id); + auto &id_info = user_.ids.at(id); + CreateTaskRunner(subflow, id, id_info); } // NOTE, Do not call detach otherwise this will fail @@ -165,41 +165,45 @@ void CustomGenerator::BuildGenerate() { } } -tf::Task CustomGenerator::CreateTaskRunner(tf::Subflow &subflow, - const std::string &id) { - return subflow.emplace([&, id]() { +tf::Task CustomGenerator::CreateTaskRunner( + tf::Subflow &subflow, const std::string &id, + UserCustomGeneratorSchema::UserIdInfo &id_info) { + auto task = subflow.emplace([&, id]() { if (env::get_task_state() != env::TaskState::SUCCESS) { return; } try { - TaskRunner(id); + id_info.ConvertToInternal(); + TaskRunner(id, id_info); } catch (...) { env::set_task_state(env::TaskState::FAILURE); } }); + task.name(id); + return task; } -void CustomGenerator::TaskRunner(const std::string &id) { - // Convert to internal - user_.ids.at(id).ConvertToInternal(); - +// TODO, 2 problems with this +// Find a way for TaskRunner (subflow to send data back to taskflow) +void CustomGenerator::TaskRunner( + const std::string &id, + const UserCustomGeneratorSchema::UserIdInfo &id_info) { // Compute runnable bool run = comparator_.IsIdAdded(id) ? true : comparator_.IsChanged(id); // Invoke generator callback - const auto ¤t_id_info = user_.ids.at(id); if (run) { dirty_ = true; - const auto input_paths = current_id_info.inputs.GetPaths(); - const auto &output_paths = current_id_info.outputs.GetPaths(); - CustomGeneratorContext ctx(command_, input_paths, output_paths, - current_id_info.userblob); - bool success = current_id_info.generate_cb(ctx); + const auto input_paths = id_info.inputs.GetPaths(); + CustomGeneratorContext ctx(command_, input_paths, + id_info.outputs.GetPaths(), id_info.userblob); + + bool success = id_info.generate_cb(ctx); env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id)); } std::scoped_lock guard(success_schema_mutex_); - success_schema_.try_emplace(id, current_id_info); + success_schema_.try_emplace(id, id_info); } } // namespace buildcc From 388dfedb57f0f92d25ba332dfe9d3144a5dc0e9e Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 2 Dec 2022 09:34:20 -0800 Subject: [PATCH 07/12] Added CustomGeneratorFunctor --- .../src/custom_generator/custom_generator.cpp | 91 ++++++++++++++----- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index ee396aae..b9f81c1f 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -100,6 +100,56 @@ void CustomGenerator::Initialize() { tf_.name(name_); } +struct TaskState { + bool should_run{false}; + bool run_success{false}; +}; + +struct CustomGeneratorFunctor { + CustomGeneratorFunctor(const std::string &id, + UserCustomGeneratorSchema::UserIdInfo &id_info, + const Comparator &comparator, + const env::Command &command, TaskState &state) + : id_(id), id_info_(id_info), comparator_(comparator), command_(command), + state_(state) {} + + void operator()() { + if (env::get_task_state() != env::TaskState::SUCCESS) { + return; + } + try { + id_info_.ConvertToInternal(); + // Compute runnable + state_.should_run = + comparator_.IsIdAdded(id_) ? true : comparator_.IsChanged(id_); + + // Invoke generator callback + if (state_.should_run) { + const auto input_paths = id_info_.inputs.GetPaths(); + CustomGeneratorContext ctx(command_, input_paths, + id_info_.outputs.GetPaths(), + id_info_.userblob); + + bool success = id_info_.generate_cb(ctx); + env::assert_fatal(success, + fmt::format("Generate Cb failed for id {}", id_)); + } + state_.run_success = true; + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + } + +private: + const std::string &id_; + UserCustomGeneratorSchema::UserIdInfo &id_info_; + + const Comparator &comparator_; + const env::Command &command_; + + TaskState &state_; +}; + void CustomGenerator::GenerateTask() { tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { if (env::get_task_state() != env::TaskState::SUCCESS) { @@ -110,25 +160,38 @@ void CustomGenerator::GenerateTask() { // Selected ids for build BuildGenerate(); + std::unordered_map states; + // Create runner for each added/updated id for (const auto &id : comparator_.GetAddedIds()) { + states.try_emplace(id, TaskState()); auto &id_info = user_.ids.at(id); - CreateTaskRunner(subflow, id, id_info); + CustomGeneratorFunctor functor(id, id_info, comparator_, command_, + states.at(id)); + subflow.emplace(functor).name(id); } for (const auto &id : comparator_.GetCheckLaterIds()) { + states.try_emplace(id, TaskState()); auto &id_info = user_.ids.at(id); - CreateTaskRunner(subflow, id, id_info); + CustomGeneratorFunctor functor(id, id_info, comparator_, command_, + states.at(id)); + subflow.emplace(functor).name(id); } // NOTE, Do not call detach otherwise this will fail subflow.join(); + UserCustomGeneratorSchema user_final_schema; + for (const auto &[id, state] : states) { + dirty_ = dirty_ || state.should_run; + if (state.run_success) { + user_final_schema.ids.try_emplace(id, user_.ids.at(id)); + } + } + // Store if (dirty_) { - UserCustomGeneratorSchema user_final_schema; - user_final_schema.ids.insert(success_schema_.begin(), - success_schema_.end()); user_final_schema.ConvertToInternal(); serialization_.UpdateStore(user_final_schema); @@ -165,24 +228,6 @@ void CustomGenerator::BuildGenerate() { } } -tf::Task CustomGenerator::CreateTaskRunner( - tf::Subflow &subflow, const std::string &id, - UserCustomGeneratorSchema::UserIdInfo &id_info) { - auto task = subflow.emplace([&, id]() { - if (env::get_task_state() != env::TaskState::SUCCESS) { - return; - } - try { - id_info.ConvertToInternal(); - TaskRunner(id, id_info); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }); - task.name(id); - return task; -} - // TODO, 2 problems with this // Find a way for TaskRunner (subflow to send data back to taskflow) void CustomGenerator::TaskRunner( From 099b8420a183b15e436d08b8514794b4357c7819 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 2 Dec 2022 09:35:25 -0800 Subject: [PATCH 08/12] Renamed to TaskFunctor --- .../src/custom_generator/custom_generator.cpp | 106 +++++++++--------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index b9f81c1f..f6aaec37 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -28,6 +28,56 @@ constexpr const char *const kCurrentBuildDirName = "current_build_dir"; namespace buildcc { +struct TaskState { + bool should_run{false}; + bool run_success{false}; +}; + +struct TaskFunctor { + TaskFunctor(const std::string &id, + UserCustomGeneratorSchema::UserIdInfo &id_info, + const Comparator &comparator, const env::Command &command, + TaskState &state) + : id_(id), id_info_(id_info), comparator_(comparator), command_(command), + state_(state) {} + + void operator()() { + if (env::get_task_state() != env::TaskState::SUCCESS) { + return; + } + try { + id_info_.ConvertToInternal(); + // Compute runnable + state_.should_run = + comparator_.IsIdAdded(id_) ? true : comparator_.IsChanged(id_); + + // Invoke generator callback + if (state_.should_run) { + const auto input_paths = id_info_.inputs.GetPaths(); + CustomGeneratorContext ctx(command_, input_paths, + id_info_.outputs.GetPaths(), + id_info_.userblob); + + bool success = id_info_.generate_cb(ctx); + env::assert_fatal(success, + fmt::format("Generate Cb failed for id {}", id_)); + } + state_.run_success = true; + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + } + +private: + const std::string &id_; + UserCustomGeneratorSchema::UserIdInfo &id_info_; + + const Comparator &comparator_; + const env::Command &command_; + + TaskState &state_; +}; + void CustomGenerator::AddPattern(const std::string &identifier, const std::string &pattern) { command_.AddDefaultArgument(identifier, command_.Construct(pattern)); @@ -100,56 +150,6 @@ void CustomGenerator::Initialize() { tf_.name(name_); } -struct TaskState { - bool should_run{false}; - bool run_success{false}; -}; - -struct CustomGeneratorFunctor { - CustomGeneratorFunctor(const std::string &id, - UserCustomGeneratorSchema::UserIdInfo &id_info, - const Comparator &comparator, - const env::Command &command, TaskState &state) - : id_(id), id_info_(id_info), comparator_(comparator), command_(command), - state_(state) {} - - void operator()() { - if (env::get_task_state() != env::TaskState::SUCCESS) { - return; - } - try { - id_info_.ConvertToInternal(); - // Compute runnable - state_.should_run = - comparator_.IsIdAdded(id_) ? true : comparator_.IsChanged(id_); - - // Invoke generator callback - if (state_.should_run) { - const auto input_paths = id_info_.inputs.GetPaths(); - CustomGeneratorContext ctx(command_, input_paths, - id_info_.outputs.GetPaths(), - id_info_.userblob); - - bool success = id_info_.generate_cb(ctx); - env::assert_fatal(success, - fmt::format("Generate Cb failed for id {}", id_)); - } - state_.run_success = true; - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - } - -private: - const std::string &id_; - UserCustomGeneratorSchema::UserIdInfo &id_info_; - - const Comparator &comparator_; - const env::Command &command_; - - TaskState &state_; -}; - void CustomGenerator::GenerateTask() { tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { if (env::get_task_state() != env::TaskState::SUCCESS) { @@ -166,16 +166,14 @@ void CustomGenerator::GenerateTask() { for (const auto &id : comparator_.GetAddedIds()) { states.try_emplace(id, TaskState()); auto &id_info = user_.ids.at(id); - CustomGeneratorFunctor functor(id, id_info, comparator_, command_, - states.at(id)); + TaskFunctor functor(id, id_info, comparator_, command_, states.at(id)); subflow.emplace(functor).name(id); } for (const auto &id : comparator_.GetCheckLaterIds()) { states.try_emplace(id, TaskState()); auto &id_info = user_.ids.at(id); - CustomGeneratorFunctor functor(id, id_info, comparator_, command_, - states.at(id)); + TaskFunctor functor(id, id_info, comparator_, command_, states.at(id)); subflow.emplace(functor).name(id); } From c2d553abf0aa7e8156c75db2b70f0de6c80e6c17 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 2 Dec 2022 09:42:54 -0800 Subject: [PATCH 09/12] Removed CreateTaskRunner and TaskRunner --- .../target/include/target/custom_generator.h | 5 ---- .../src/custom_generator/custom_generator.cpp | 23 ------------------- 2 files changed, 28 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index f7ef0d02..e3065335 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -196,11 +196,6 @@ class CustomGenerator : public internal::BuilderInterface { private: void Initialize(); - tf::Task CreateTaskRunner(tf::Subflow &subflow, const std::string &id, - UserCustomGeneratorSchema::UserIdInfo &id_info); - void TaskRunner(const std::string &id, - const UserCustomGeneratorSchema::UserIdInfo &id_info); - void GenerateTask(); void BuildGenerate(); diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index f6aaec37..c5434819 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -226,27 +226,4 @@ void CustomGenerator::BuildGenerate() { } } -// TODO, 2 problems with this -// Find a way for TaskRunner (subflow to send data back to taskflow) -void CustomGenerator::TaskRunner( - const std::string &id, - const UserCustomGeneratorSchema::UserIdInfo &id_info) { - // Compute runnable - bool run = comparator_.IsIdAdded(id) ? true : comparator_.IsChanged(id); - - // Invoke generator callback - if (run) { - dirty_ = true; - const auto input_paths = id_info.inputs.GetPaths(); - CustomGeneratorContext ctx(command_, input_paths, - id_info.outputs.GetPaths(), id_info.userblob); - - bool success = id_info.generate_cb(ctx); - env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id)); - } - - std::scoped_lock guard(success_schema_mutex_); - success_schema_.try_emplace(id, id_info); -} - } // namespace buildcc From c75d046186e15c0eb46fca7bb1c8c7ce3518d8b1 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 2 Dec 2022 10:30:23 -0800 Subject: [PATCH 10/12] Shifted Comparator internally --- .../target/include/target/custom_generator.h | 17 +---- .../src/custom_generator/custom_generator.cpp | 68 ++++++++++--------- 2 files changed, 40 insertions(+), 45 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index e3065335..da20fb14 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -78,7 +78,7 @@ struct Comparator { } } - void CompareIds() { + void CompareAndAddIds() { const auto &prev_ids = loaded_schema_.internal_ids; const auto &curr_ids = current_schema_.ids; @@ -147,8 +147,8 @@ class CustomGenerator : public internal::BuilderInterface { CustomGenerator(const std::string &name, const TargetEnv &env) : name_(name), env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), - serialization_(env_.GetTargetBuildDir() / fmt::format("{}.json", name)), - comparator_(serialization_.GetLoad(), user_) { + serialization_(env_.GetTargetBuildDir() / + fmt::format("{}.json", name)) { Initialize(); } virtual ~CustomGenerator() = default; @@ -195,9 +195,7 @@ class CustomGenerator : public internal::BuilderInterface { private: void Initialize(); - void GenerateTask(); - void BuildGenerate(); // Recheck states void IdRemoved(); @@ -218,15 +216,6 @@ class CustomGenerator : public internal::BuilderInterface { // Internal env::Command command_; - - // Comparator - // TODO, Shift internally - Comparator comparator_; - - // TODO, Shift internally - std::mutex success_schema_mutex_; - std::unordered_map - success_schema_; }; } // namespace buildcc diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index c5434819..82cdea0e 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -38,7 +38,7 @@ struct TaskFunctor { UserCustomGeneratorSchema::UserIdInfo &id_info, const Comparator &comparator, const env::Command &command, TaskState &state) - : id_(id), id_info_(id_info), comparator_(comparator), command_(command), + : id_(id), id_info_(id_info), comparator(comparator), command_(command), state_(state) {} void operator()() { @@ -49,7 +49,7 @@ struct TaskFunctor { id_info_.ConvertToInternal(); // Compute runnable state_.should_run = - comparator_.IsIdAdded(id_) ? true : comparator_.IsChanged(id_); + comparator.IsIdAdded(id_) ? true : comparator.IsChanged(id_); // Invoke generator callback if (state_.should_run) { @@ -72,12 +72,38 @@ struct TaskFunctor { const std::string &id_; UserCustomGeneratorSchema::UserIdInfo &id_info_; - const Comparator &comparator_; + const Comparator &comparator; const env::Command &command_; TaskState &state_; }; +bool ComputeBuild(const internal::CustomGeneratorSerialization &serialization, + Comparator &comparator, + std::function &&id_removed_cb, + std::function &&id_added_cb) { + bool build = false; + if (!serialization.IsLoaded()) { + comparator.AddAllIds(); + build = true; + } else { + comparator.CompareAndAddIds(); + const bool is_removed = !comparator.GetRemovedIds().empty(); + const bool is_added = !comparator.GetAddedIds().empty(); + build = is_removed || is_added; + + if (is_removed) { + id_removed_cb(); + } + + for (const auto &id : comparator.GetAddedIds()) { + (void)id; + id_added_cb(); + } + } + return build; +} + void CustomGenerator::AddPattern(const std::string &identifier, const std::string &pattern) { command_.AddDefaultArgument(identifier, command_.Construct(pattern)); @@ -158,22 +184,25 @@ void CustomGenerator::GenerateTask() { try { // Selected ids for build - BuildGenerate(); + Comparator comparator(serialization_.GetLoad(), user_); + dirty_ = ComputeBuild( + serialization_, comparator, [this]() { IdRemoved(); }, + [this]() { IdAdded(); }); std::unordered_map states; // Create runner for each added/updated id - for (const auto &id : comparator_.GetAddedIds()) { + for (const auto &id : comparator.GetAddedIds()) { states.try_emplace(id, TaskState()); auto &id_info = user_.ids.at(id); - TaskFunctor functor(id, id_info, comparator_, command_, states.at(id)); + TaskFunctor functor(id, id_info, comparator, command_, states.at(id)); subflow.emplace(functor).name(id); } - for (const auto &id : comparator_.GetCheckLaterIds()) { + for (const auto &id : comparator.GetCheckLaterIds()) { states.try_emplace(id, TaskState()); auto &id_info = user_.ids.at(id); - TaskFunctor functor(id, id_info, comparator_, command_, states.at(id)); + TaskFunctor functor(id, id_info, comparator, command_, states.at(id)); subflow.emplace(functor).name(id); } @@ -203,27 +232,4 @@ void CustomGenerator::GenerateTask() { generate_task.name(kGenerateTaskName); } -void CustomGenerator::BuildGenerate() { - if (!serialization_.IsLoaded()) { - comparator_.AddAllIds(); - dirty_ = true; - } else { - // For IDS - comparator_.CompareIds(); - - const bool is_removed = !comparator_.GetRemovedIds().empty(); - const bool is_added = !comparator_.GetAddedIds().empty(); - dirty_ = is_removed || is_added; - - if (is_removed) { - IdRemoved(); - } - - for (const auto &id : comparator_.GetAddedIds()) { - (void)id; - IdAdded(); - } - } -} - } // namespace buildcc From bc26b1f557d929c02eb40a98be6890096e3d8c77 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 2 Dec 2022 10:53:53 -0800 Subject: [PATCH 11/12] Shifted Comparator implementation to custom_generator.cpp file --- .../target/include/target/custom_generator.h | 82 ------------------- .../src/custom_generator/custom_generator.cpp | 82 +++++++++++++++++++ 2 files changed, 82 insertions(+), 82 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index da20fb14..a9dec033 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -60,88 +60,6 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { std::unordered_map ids; }; -struct Comparator { - Comparator(const buildcc::internal::CustomGeneratorSchema &loaded, - const buildcc::UserCustomGeneratorSchema ¤t) - : loaded_schema_(loaded), current_schema_(current) {} - - enum class State { - kRemoved, - kAdded, - kCheckLater, - }; - - void AddAllIds() { - const auto &curr_ids = current_schema_.ids; - for (const auto &[id, _] : curr_ids) { - id_state_info_.at(State::kAdded).insert(id); - } - } - - void CompareAndAddIds() { - const auto &prev_ids = loaded_schema_.internal_ids; - const auto &curr_ids = current_schema_.ids; - - for (const auto &[prev_id, _] : prev_ids) { - if (curr_ids.find(prev_id) == curr_ids.end()) { - // Id Removed condition, previous id is not present in the current run - id_state_info_.at(State::kRemoved).insert(prev_id); - } - } - - for (const auto &[curr_id, _] : curr_ids) { - if (prev_ids.find(curr_id) == prev_ids.end()) { - // Id Added condition - id_state_info_.at(State::kAdded).insert(curr_id); - } else { - // Id Check Later condition - id_state_info_.at(State::kCheckLater).insert(curr_id); - } - } - } - - bool IsChanged(const std::string &id) const { - const auto &previous_id_info = loaded_schema_.internal_ids.at(id); - const auto ¤t_id_info = current_schema_.ids.at(id); - - bool changed = !previous_id_info.inputs.IsEqual(current_id_info.inputs) || - !previous_id_info.outputs.IsEqual(current_id_info.outputs); - if (!changed && current_id_info.blob_handler != nullptr) { - // We only check blob handler if not changed by inputs/outputs - // Checking blob_handler could be expensive so this optimization is made - // to run only when changed == false - changed = current_id_info.blob_handler->CheckChanged( - previous_id_info.userblob, current_id_info.userblob); - } - return changed; - } - - const std::unordered_set &GetRemovedIds() const { - return id_state_info_.at(State::kRemoved); - } - - const std::unordered_set &GetAddedIds() const { - return id_state_info_.at(State::kAdded); - } - - const std::unordered_set &GetCheckLaterIds() const { - return id_state_info_.at(State::kCheckLater); - } - - bool IsIdAdded(const std::string &id) const { - return id_state_info_.at(State::kAdded).count(id) == 1; - } - -private: - const buildcc::internal::CustomGeneratorSchema &loaded_schema_; - const buildcc::UserCustomGeneratorSchema ¤t_schema_; - std::unordered_map> id_state_info_{ - {State::kRemoved, std::unordered_set()}, - {State::kAdded, std::unordered_set()}, - {State::kCheckLater, std::unordered_set()}, - }; -}; - class CustomGenerator : public internal::BuilderInterface { public: CustomGenerator(const std::string &name, const TargetEnv &env) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 82cdea0e..9314b590 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -28,6 +28,88 @@ constexpr const char *const kCurrentBuildDirName = "current_build_dir"; namespace buildcc { +struct Comparator { + Comparator(const buildcc::internal::CustomGeneratorSchema &loaded, + const buildcc::UserCustomGeneratorSchema ¤t) + : loaded_schema_(loaded), current_schema_(current) {} + + enum class State { + kRemoved, + kAdded, + kCheckLater, + }; + + void AddAllIds() { + const auto &curr_ids = current_schema_.ids; + for (const auto &[id, _] : curr_ids) { + id_state_info_.at(State::kAdded).insert(id); + } + } + + void CompareAndAddIds() { + const auto &prev_ids = loaded_schema_.internal_ids; + const auto &curr_ids = current_schema_.ids; + + for (const auto &[prev_id, _] : prev_ids) { + if (curr_ids.find(prev_id) == curr_ids.end()) { + // Id Removed condition, previous id is not present in the current run + id_state_info_.at(State::kRemoved).insert(prev_id); + } + } + + for (const auto &[curr_id, _] : curr_ids) { + if (prev_ids.find(curr_id) == prev_ids.end()) { + // Id Added condition + id_state_info_.at(State::kAdded).insert(curr_id); + } else { + // Id Check Later condition + id_state_info_.at(State::kCheckLater).insert(curr_id); + } + } + } + + bool IsChanged(const std::string &id) const { + const auto &previous_id_info = loaded_schema_.internal_ids.at(id); + const auto ¤t_id_info = current_schema_.ids.at(id); + + bool changed = !previous_id_info.inputs.IsEqual(current_id_info.inputs) || + !previous_id_info.outputs.IsEqual(current_id_info.outputs); + if (!changed && current_id_info.blob_handler != nullptr) { + // We only check blob handler if not changed by inputs/outputs + // Checking blob_handler could be expensive so this optimization is made + // to run only when changed == false + changed = current_id_info.blob_handler->CheckChanged( + previous_id_info.userblob, current_id_info.userblob); + } + return changed; + } + + const std::unordered_set &GetRemovedIds() const { + return id_state_info_.at(State::kRemoved); + } + + const std::unordered_set &GetAddedIds() const { + return id_state_info_.at(State::kAdded); + } + + const std::unordered_set &GetCheckLaterIds() const { + return id_state_info_.at(State::kCheckLater); + } + + bool IsIdAdded(const std::string &id) const { + return id_state_info_.at(State::kAdded).count(id) == 1; + } + +private: + const buildcc::internal::CustomGeneratorSchema &loaded_schema_; + const buildcc::UserCustomGeneratorSchema ¤t_schema_; + std::unordered_map> id_state_info_{ + {State::kRemoved, std::unordered_set()}, + {State::kAdded, std::unordered_set()}, + {State::kCheckLater, std::unordered_set()}, + }; +}; + struct TaskState { bool should_run{false}; bool run_success{false}; From f6f1aaf9cb3c92153afcb4b45ebd6e48c0a3f89e Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 2 Dec 2022 11:15:24 -0800 Subject: [PATCH 12/12] Updated comparator --- .../src/custom_generator/custom_generator.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 9314b590..138bb9f3 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -29,9 +29,9 @@ constexpr const char *const kCurrentBuildDirName = "current_build_dir"; namespace buildcc { struct Comparator { - Comparator(const buildcc::internal::CustomGeneratorSchema &loaded, - const buildcc::UserCustomGeneratorSchema ¤t) - : loaded_schema_(loaded), current_schema_(current) {} + Comparator(const internal::CustomGeneratorSchema &loaded, + const UserCustomGeneratorSchema ¤t) + : loaded_(loaded), current_(current) {} enum class State { kRemoved, @@ -40,15 +40,15 @@ struct Comparator { }; void AddAllIds() { - const auto &curr_ids = current_schema_.ids; + const auto &curr_ids = current_.ids; for (const auto &[id, _] : curr_ids) { id_state_info_.at(State::kAdded).insert(id); } } void CompareAndAddIds() { - const auto &prev_ids = loaded_schema_.internal_ids; - const auto &curr_ids = current_schema_.ids; + const auto &prev_ids = loaded_.internal_ids; + const auto &curr_ids = current_.ids; for (const auto &[prev_id, _] : prev_ids) { if (curr_ids.find(prev_id) == curr_ids.end()) { @@ -69,8 +69,8 @@ struct Comparator { } bool IsChanged(const std::string &id) const { - const auto &previous_id_info = loaded_schema_.internal_ids.at(id); - const auto ¤t_id_info = current_schema_.ids.at(id); + const auto &previous_id_info = loaded_.internal_ids.at(id); + const auto ¤t_id_info = current_.ids.at(id); bool changed = !previous_id_info.inputs.IsEqual(current_id_info.inputs) || !previous_id_info.outputs.IsEqual(current_id_info.outputs); @@ -101,8 +101,8 @@ struct Comparator { } private: - const buildcc::internal::CustomGeneratorSchema &loaded_schema_; - const buildcc::UserCustomGeneratorSchema ¤t_schema_; + const buildcc::internal::CustomGeneratorSchema &loaded_; + const buildcc::UserCustomGeneratorSchema ¤t_; std::unordered_map> id_state_info_{ {State::kRemoved, std::unordered_set()}, {State::kAdded, std::unordered_set()},