From 3d832f685281a1befe726d46753c292971d7bd7d Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 00:22:29 -0700 Subject: [PATCH 01/18] Updated custom_generator implementation for groups --- .../target/include/target/custom_generator.h | 23 ++- .../src/custom_generator/custom_generator.cpp | 139 ++++++++++++------ 2 files changed, 116 insertions(+), 46 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 76e64576..155b229b 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -53,7 +53,7 @@ class CustomGeneratorContext { // clang-format off typedef std::function GenerateCb; -typedef std::function &)> DependencyCb; +typedef std::function &&)> DependencyCb; // clang-format on class CustomBlobHandler { @@ -136,6 +136,10 @@ class CustomGenerator : public internal::BuilderInterface { const GenerateCb &generate_cb, std::shared_ptr blob_handler = nullptr); + void AddGroup(const std::string &group_id, + std::initializer_list ids, + const DependencyCb &dependency_cb = DependencyCb()); + // Callbacks /** * @brief Setup dependencies between Tasks using their `id` @@ -168,12 +172,13 @@ class CustomGenerator : public internal::BuilderInterface { private: void Initialize(); - template void TaskRunner(const std::string &id); + void TaskRunner(bool run, const std::string &id); + tf::Task CreateTaskRunner(tf::Subflow &subflow, bool build, + const std::string &id); void GenerateTask(); - void BuildGenerate( - std::unordered_map &gen_selected_map, - std::unordered_map &dummy_gen_selected_map); + void BuildGenerate(std::unordered_set &gen_selected_ids, + std::unordered_set &dummy_gen_selected_ids); // Recheck states void IdRemoved(); @@ -183,6 +188,12 @@ class CustomGenerator : public internal::BuilderInterface { protected: env::Command command_; +private: + struct GroupMetadata { + std::vector ids; + DependencyCb dependency_cb; + }; + private: std::string name_; TargetEnv env_; @@ -190,6 +201,8 @@ class CustomGenerator : public internal::BuilderInterface { // Serialization UserCustomGeneratorSchema user_; + std::unordered_map grouped_ids_; + std::unordered_set ungrouped_ids_; std::mutex success_schema_mutex_; std::unordered_map success_schema_; diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 466d43dd..9e6a7d3e 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -68,6 +68,28 @@ void CustomGenerator::AddGenInfo( schema.generate_cb = generate_cb; schema.blob_handler = std::move(blob_handler); user_.gen_info_map.emplace(id, std::move(schema)); + ungrouped_ids_.emplace(id); +} + +void CustomGenerator::AddGroup(const std::string &group_id, + std::initializer_list ids, + const DependencyCb &dependency_cb) { + // Verify that the ids exist + // Remove those ids from ungrouped_ids + for (const auto &id : ids) { + env::assert_fatal(user_.gen_info_map.find(id) != user_.gen_info_map.end(), + fmt::format("Id '{}' is not found", id)); + ungrouped_ids_.erase(id); + } + + env::assert_fatal(grouped_ids_.find(group_id) == grouped_ids_.end(), + fmt::format("Group Id '{}' duplicate found", group_id)); + + // Group map is used to group similar ids in a single subflow + GroupMetadata group_metadata; + group_metadata.ids = ids; + group_metadata.dependency_cb = dependency_cb; + grouped_ids_.emplace(group_id, std::move(group_metadata)); } void CustomGenerator::AddDependencyCb(const DependencyCb &dependency_cb) { @@ -102,10 +124,12 @@ void CustomGenerator::Initialize() { } void CustomGenerator::BuildGenerate( - std::unordered_map &gen_selected_map, - std::unordered_map &dummy_gen_selected_map) { + std::unordered_set &gen_selected_ids, + std::unordered_set &dummy_gen_selected_ids) { if (!serialization_.IsLoaded()) { - gen_selected_map = user_.gen_info_map; + std::for_each( + user_.gen_info_map.begin(), user_.gen_info_map.end(), + [&](const auto &iter) { gen_selected_ids.insert(iter.first); }); dirty_ = true; } else { // DONE, Conditionally select internal_gen_info_map depending on what has @@ -136,13 +160,13 @@ void CustomGenerator::BuildGenerate( if (prev_gen_info_map.find(id) == prev_gen_info_map.end()) { // MAP ADDED condition IdAdded(); - gen_selected_map.emplace(curr_miter.first, curr_miter.second); + gen_selected_ids.insert(curr_miter.first); dirty_ = true; } else { // MAP UPDATED condition (*checked later) // This is because tasks can have dependencies amongst each other we can // compute task level rebuilds later - dummy_gen_selected_map.emplace(curr_miter.first, curr_miter.second); + dummy_gen_selected_ids.insert(curr_miter.first); } } } @@ -155,46 +179,63 @@ void CustomGenerator::GenerateTask() { } try { - std::unordered_map selected_user_schema; - std::unordered_map dummy_selected_user_schema; - BuildGenerate(selected_user_schema, dummy_selected_user_schema); - - std::unordered_map task_map; - // Create task for selected schema - for (const auto &selected_miter : selected_user_schema) { - const auto &id = selected_miter.first; - // const auto ¤t_info = selected_miter.second; - tf::Task task = subflow - .emplace([&]() { - try { - TaskRunner(id); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }) - .name(id); - task_map.emplace(id, task); + std::unordered_map registered_tasks; + + // Selected ids for build + std::unordered_set selected_ids; + std::unordered_set dummy_selected_ids; + BuildGenerate(selected_ids, dummy_selected_ids); + + // Grouped tasks + for (const auto &group_iter : grouped_ids_) { + const auto &group_id = group_iter.first; + const auto &group_metadata = group_iter.second; + auto group_task = subflow.emplace([&](tf::Subflow &s) { + std::unordered_map reg_tasks; + + if (env::get_task_state() != env::TaskState::SUCCESS) { + return; + } + + for (const auto &id : group_metadata.ids) { + bool build = selected_ids.count(id) == 1; + auto task = CreateTaskRunner(s, build, id); + task.name(id); + reg_tasks.emplace(id, task); + } + + // Dependency callback + if (group_metadata.dependency_cb) { + try { + group_metadata.dependency_cb(std::move(reg_tasks)); + } catch (...) { + env::log_critical( + __FUNCTION__, + fmt::format("Dependency callback failed for group id {}", + group_id)); + env::set_task_state(env::TaskState::FAILURE); + } + } + + // NOTE, Do not call detach otherwise this will fail + s.join(); + }); + group_task.name(group_id); + registered_tasks.emplace(group_id, group_task); } - for (auto &dummy_selected_miter : dummy_selected_user_schema) { - const auto &id = dummy_selected_miter.first; - // const auto ¤t_info = dummy_selected_miter.second; - tf::Task task = subflow - .emplace([&]() { - try { - TaskRunner(id); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }) - .name(id); - task_map.emplace(id, task); + // Ungrouped tasks + for (const auto &id : ungrouped_ids_) { + bool build = selected_ids.count(id) == 1; + auto task = CreateTaskRunner(subflow, build, id); + task.name(id); + registered_tasks.emplace(id, task); } - // Dependencies between ids + // Dependencies between tasks if (dependency_cb_) { try { - dependency_cb_(task_map); + dependency_cb_(std::move(registered_tasks)); } catch (...) { env::log_critical(__FUNCTION__, "Dependency callback failed"); env::set_task_state(env::TaskState::FAILURE); @@ -224,7 +265,23 @@ void CustomGenerator::GenerateTask() { generate_task.name(kGenerateTaskName); } -template void CustomGenerator::TaskRunner(const std::string &id) { +tf::Task CustomGenerator::CreateTaskRunner(tf::Subflow &subflow, bool build, + const std::string &id) { + return subflow.emplace([&, build, id]() { + if (env::get_task_state() != env::TaskState::SUCCESS) { + return; + } + try { + TaskRunner(build, id); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }); +} + +void CustomGenerator::TaskRunner(bool run, const std::string &id) { + env::log_critical(__FUNCTION__, id); + // Convert { auto ¤t_gen_info = user_.gen_info_map.at(id); @@ -239,7 +296,7 @@ template void CustomGenerator::TaskRunner(const std::string &id) { // Run const auto ¤t_info = user_.gen_info_map.at(id); bool rerun = false; - if constexpr (run) { + if (run) { rerun = true; } else { const auto &previous_info = From 30c8336fbc6189334fb6cd3589cd39017f3715f5 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 00:23:34 -0700 Subject: [PATCH 02/18] Updated test_custom_generator, minor change --- .../lib/target/test/target/test_custom_generator.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 5efa7e6f..2d3f43ef 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -183,7 +183,7 @@ static bool Dep2Cb(const buildcc::CustomGeneratorContext &ctx) { return buildcc::env::Command::Execute(""); } -static void DependencyCb(std::unordered_map &task_map) { +static void DependencyCb(std::unordered_map &&task_map) { task_map.at("id1").precede(task_map.at("id2")); } @@ -417,8 +417,10 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Basic) { mock().expectOneCall("RealGenerateCb"); buildcc::env::m::CommandExpect_Execute(1, false); - mock().expectOneCall("RealGenerateCb"); - buildcc::env::m::CommandExpect_Execute(1, true); + + // Since there is an error above, second command does not execute (note, + // this is the behaviour in a single thread that is why we can + // check sequentially) buildcc::m::CustomGeneratorRunner(cgen); CHECK_TRUE(buildcc::env::get_task_state() == @@ -427,7 +429,7 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Basic) { buildcc::internal::CustomGeneratorSerialization serialization( cgen.GetBinaryPath()); CHECK_TRUE(serialization.LoadFromFile()); - CHECK_EQUAL(serialization.GetLoad().internal_gen_info_map.size(), 1); + CHECK_EQUAL(serialization.GetLoad().internal_gen_info_map.size(), 0); fs::remove_all(cgen.GetBinaryPath()); } From eed9491571967e94c733fdc93eaae25077174830 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 02:40:00 -0700 Subject: [PATCH 03/18] Updated test_custom_generator with group tests --- .../test/target/test_custom_generator.cpp | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 2d3f43ef..478ce981 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -85,6 +85,97 @@ TEST(CustomGeneratorTestGroup, Basic_Failure) { CHECK_EQUAL(internal_map.size(), 1); } +TEST(CustomGeneratorTestGroup, Basic_Group) { + buildcc::CustomGenerator cgen("basic_group", ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, BasicGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, + BasicGenerateCb); + cgen.AddGroup("grouped_id1_and_id2", {"id1", "id2"}); + cgen.Build(); + + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + buildcc::m::CustomGeneratorRunner(cgen); + + // Serialization check + { + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + + const auto &internal_map = serialization.GetLoad().internal_gen_info_map; + CHECK_EQUAL(internal_map.size(), 2); + const auto &id1_info = internal_map.at("id1"); + CHECK_EQUAL(id1_info.internal_inputs.size(), 1); + CHECK_EQUAL(id1_info.outputs.size(), 1); + + const auto &id2_info = internal_map.at("id2"); + CHECK_EQUAL(id2_info.internal_inputs.size(), 1); + CHECK_EQUAL(id2_info.outputs.size(), 0); + } +} + +TEST(CustomGeneratorTestGroup, Basic_Group_Dependency) { + buildcc::CustomGenerator cgen("basic_group_dependency", ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, BasicGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, + BasicGenerateCb); + cgen.AddGroup("grouped_id1_and_id2", {"id1", "id2"}, [](auto &&task_map) { + task_map.at("id1").precede(task_map.at("id2")); + }); + cgen.Build(); + + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + buildcc::m::CustomGeneratorRunner(cgen); + + // Serialization check + { + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + + const auto &internal_map = serialization.GetLoad().internal_gen_info_map; + CHECK_EQUAL(internal_map.size(), 2); + const auto &id1_info = internal_map.at("id1"); + CHECK_EQUAL(id1_info.internal_inputs.size(), 1); + CHECK_EQUAL(id1_info.outputs.size(), 1); + + const auto &id2_info = internal_map.at("id2"); + CHECK_EQUAL(id2_info.internal_inputs.size(), 1); + CHECK_EQUAL(id2_info.outputs.size(), 0); + } +} + +TEST(CustomGeneratorTestGroup, Basic_Group_DependencyFailure) { + buildcc::CustomGenerator cgen("basic_group_dependency_failure", ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, BasicGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, + BasicGenerateCb); + cgen.AddGroup("grouped_id1_and_id2", {"id1", "id2"}, [](auto &&task_map) { + task_map.at("id1").precede(task_map.at("id2")); + buildcc::env::assert_fatal("Failure"); + }); + cgen.Build(); + + buildcc::m::CustomGeneratorRunner(cgen); + + // Serialization check + { + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + + const auto &internal_map = serialization.GetLoad().internal_gen_info_map; + CHECK_EQUAL(internal_map.size(), 0); + } + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); +} + TEST(CustomGeneratorTestGroup, DefaultArgumentUsage) { buildcc::CustomGenerator cgen("default_argument_usage", ""); cgen.AddPatterns({ From 338e3560d2bad0f22ee67671774f6f30aa33112c Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 03:04:47 -0700 Subject: [PATCH 04/18] Updated test_custom_generator --- .../test/target/test_custom_generator.cpp | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 478ce981..6ed2c89d 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -176,6 +176,88 @@ TEST(CustomGeneratorTestGroup, Basic_Group_DependencyFailure) { CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); } +// Behaviour +// Initial: A | B -> Passes +// Changes: (GID:NEW)[A -> B] -> No rebuild triggered + +// Behaviour +// Initial: A | B -> Fails +// Changes: (GID:NEW)[A -> B] -> rebuild triggered due to previous failure + +// DONE, Make B fail because it properly depends on A +static bool rebuild_value{false}; +static bool ProperDependency1(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + mock().actualCall("ProperDependency1"); + rebuild_value = true; + return true; +} + +static bool ProperDependency2(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + mock().actualCall("ProperDependency2"); + return rebuild_value; +} + +// ProperDependency2 depends on ProperDependency1 completion +TEST(CustomGeneratorTestGroup, Basic_ProperDependency_GoodCase) { + rebuild_value = false; + + buildcc::CustomGenerator cgen("basic_proper_dependency_good_case", ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, ProperDependency1); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, + ProperDependency2); + cgen.AddDependencyCb( + [](auto &&task_map) { task_map.at("id1").precede(task_map.at("id2")); }); + cgen.Build(); + + mock().expectOneCall("ProperDependency1"); + mock().expectOneCall("ProperDependency2"); + buildcc::m::CustomGeneratorRunner(cgen); + + // Serialization check + { + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + + const auto &internal_map = serialization.GetLoad().internal_gen_info_map; + CHECK_EQUAL(internal_map.size(), 2); + } + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); +} + +// ProperDependency2 depends on ProperDependency1 completion +TEST(CustomGeneratorTestGroup, Basic_ProperDependency_BadCase) { + rebuild_value = false; + + buildcc::CustomGenerator cgen("basic_proper_dependency_bad_case", ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, ProperDependency1); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, + ProperDependency2); + cgen.AddDependencyCb( + [](auto &&task_map) { task_map.at("id2").precede(task_map.at("id1")); }); + cgen.Build(); + + mock().expectOneCall("ProperDependency2"); + buildcc::m::CustomGeneratorRunner(cgen); + + // Serialization check + { + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + + const auto &internal_map = serialization.GetLoad().internal_gen_info_map; + CHECK_EQUAL(internal_map.size(), 0); + } + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); +} + TEST(CustomGeneratorTestGroup, DefaultArgumentUsage) { buildcc::CustomGenerator cgen("default_argument_usage", ""); cgen.AddPatterns({ From 8b31d3b2e6f0daa3bee5ca7d7e0cdd1d9b864dbb Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 21:22:48 -0700 Subject: [PATCH 05/18] Updated test_custom_generator --- .../test/target/test_custom_generator.cpp | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 6ed2c89d..44dc65cf 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -176,6 +176,43 @@ TEST(CustomGeneratorTestGroup, Basic_Group_DependencyFailure) { CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); } +bool FailureCb(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + return false; +} +bool SuccessCb(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + return true; +} + +// An ungrouped task a dependency on a grouped task and fail the +// ungrouped task +TEST(CustomGeneratorTestGroup, Basic_Group_DependencyFailure2) { + buildcc::CustomGenerator cgen("basic_group_dependency_failure2", ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, FailureCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, SuccessCb); + cgen.AddGroup("grouped_id2", {"id2"}); + cgen.AddDependencyCb([&](auto &&task_map) { + task_map.at("id1").precede(task_map.at("grouped_id2")); + }); + cgen.Build(); + + buildcc::m::CustomGeneratorRunner(cgen); + + // Serialization check + { + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + + const auto &internal_map = serialization.GetLoad().internal_gen_info_map; + CHECK_EQUAL(internal_map.size(), 0); + } + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); +} + // Behaviour // Initial: A | B -> Passes // Changes: (GID:NEW)[A -> B] -> No rebuild triggered @@ -184,6 +221,25 @@ TEST(CustomGeneratorTestGroup, Basic_Group_DependencyFailure) { // Initial: A | B -> Fails // Changes: (GID:NEW)[A -> B] -> rebuild triggered due to previous failure +// ! IMPORTANT +// * NOTE, It is users responsibility to make sure that when A -> B, A's data +// change should automatically trigger B + +// For example: Say A -> B i.e B depends on A +// In a typical scenario, B would depend on A's output +// To make sure B is triggered when A changes. Make sure you use A's output in +// B's userblob. +// In this way whenever A changes, B's userblob automatically becomes "newer" +// and triggers a rebuild as well + +// Say, A gives out "rebuild = true/false" as its output +// We can use this "rebuild" variable in B's userblob +// When A runs and "rebuild" changes from false to true, During the `TaskRunner` +// we check B's userblob and automatically invoke the `CheckChanged` virtual +// call +// TODO, Create a testcase for the above scenario (Advanced_DependencyRebuild +// scenario) + // DONE, Make B fail because it properly depends on A static bool rebuild_value{false}; static bool ProperDependency1(const buildcc::CustomGeneratorContext &ctx) { From 3c66ac9b4dad04713ad22f2d6a772064259012b2 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 21:26:11 -0700 Subject: [PATCH 06/18] Updated file and template generator --- buildcc/lib/target/include/target/file_generator.h | 1 + buildcc/lib/target/include/target/template_generator.h | 1 + 2 files changed, 2 insertions(+) diff --git a/buildcc/lib/target/include/target/file_generator.h b/buildcc/lib/target/include/target/file_generator.h index 08eb1d99..27ff1946 100644 --- a/buildcc/lib/target/include/target/file_generator.h +++ b/buildcc/lib/target/include/target/file_generator.h @@ -73,6 +73,7 @@ class FileGenerator : public CustomGenerator { private: using CustomGenerator::AddDependencyCb; using CustomGenerator::AddGenInfo; + using CustomGenerator::AddGroup; using CustomGenerator::Build; private: diff --git a/buildcc/lib/target/include/target/template_generator.h b/buildcc/lib/target/include/target/template_generator.h index 2d2061a5..17898183 100644 --- a/buildcc/lib/target/include/target/template_generator.h +++ b/buildcc/lib/target/include/target/template_generator.h @@ -41,6 +41,7 @@ class TemplateGenerator : public CustomGenerator { private: using CustomGenerator::AddDependencyCb; using CustomGenerator::AddGenInfo; + using CustomGenerator::AddGroup; using CustomGenerator::Build; private: From 8c47834dec998203e945ac755d24c9f14f2c11fe Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 21:55:18 -0700 Subject: [PATCH 07/18] Updated custom_generator with code fixes --- .../src/custom_generator/custom_generator.cpp | 53 ++++++++----------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 9e6a7d3e..6003d70b 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -17,10 +17,7 @@ #include "target/custom_generator.h" namespace { -constexpr const char *const kStartGeneratorTaskName = "Start Generator"; -constexpr const char *const kEndGeneratorTaskName = "End Generator"; -constexpr const char *const kCommandTaskName = "Command"; constexpr const char *const kGenerateTaskName = "Generate"; } // namespace @@ -34,8 +31,8 @@ void CustomGenerator::AddPattern(const std::string &identifier, void CustomGenerator::AddPatterns( const std::unordered_map &pattern_map) { - for (const auto &arg_iter : pattern_map) { - AddPattern(arg_iter.first, arg_iter.second); + for (const auto &[identifier, pattern] : pattern_map) { + AddPattern(identifier, pattern); } } @@ -67,7 +64,7 @@ void CustomGenerator::AddGenInfo( } schema.generate_cb = generate_cb; schema.blob_handler = std::move(blob_handler); - user_.gen_info_map.emplace(id, std::move(schema)); + user_.gen_info_map.try_emplace(id, std::move(schema)); ungrouped_ids_.emplace(id); } @@ -89,7 +86,7 @@ void CustomGenerator::AddGroup(const std::string &group_id, GroupMetadata group_metadata; group_metadata.ids = ids; group_metadata.dependency_cb = dependency_cb; - grouped_ids_.emplace(group_id, std::move(group_metadata)); + grouped_ids_.try_emplace(group_id, std::move(group_metadata)); } void CustomGenerator::AddDependencyCb(const DependencyCb &dependency_cb) { @@ -142,8 +139,7 @@ void CustomGenerator::BuildGenerate( // curr_gen_info_map If prev_gen_info_map does not exist in // curr_gen_info_map, has been removed from existing build We need this // condition to only set the dirty_ flag - for (const auto &prev_miter : prev_gen_info_map) { - const auto &id = prev_miter.first; + for (const auto &[id, _] : prev_gen_info_map) { if (curr_gen_info_map.find(id) == curr_gen_info_map.end()) { // MAP REMOVED condition IdRemoved(); @@ -155,18 +151,17 @@ void CustomGenerator::BuildGenerate( // DONE, MAP ADDED condition Check if curr_gen_info_map exists in // prev_gen_info_map If curr_gen_info_map does not exist in // prev_gen_info_map, has been added to existing build - for (const auto &curr_miter : curr_gen_info_map) { - const auto &id = curr_miter.first; + for (const auto &[id, _] : curr_gen_info_map) { if (prev_gen_info_map.find(id) == prev_gen_info_map.end()) { // MAP ADDED condition IdAdded(); - gen_selected_ids.insert(curr_miter.first); + gen_selected_ids.insert(id); dirty_ = true; } else { // MAP UPDATED condition (*checked later) // This is because tasks can have dependencies amongst each other we can // compute task level rebuilds later - dummy_gen_selected_ids.insert(curr_miter.first); + dummy_gen_selected_ids.insert(id); } } } @@ -187,9 +182,9 @@ void CustomGenerator::GenerateTask() { BuildGenerate(selected_ids, dummy_selected_ids); // Grouped tasks - for (const auto &group_iter : grouped_ids_) { - const auto &group_id = group_iter.first; - const auto &group_metadata = group_iter.second; + for (const auto &[first, second] : grouped_ids_) { + const auto &group_id = first; + const auto &group_metadata = second; auto group_task = subflow.emplace([&](tf::Subflow &s) { std::unordered_map reg_tasks; @@ -201,7 +196,7 @@ void CustomGenerator::GenerateTask() { bool build = selected_ids.count(id) == 1; auto task = CreateTaskRunner(s, build, id); task.name(id); - reg_tasks.emplace(id, task); + reg_tasks.try_emplace(id, task); } // Dependency callback @@ -221,7 +216,7 @@ void CustomGenerator::GenerateTask() { s.join(); }); group_task.name(group_id); - registered_tasks.emplace(group_id, group_task); + registered_tasks.try_emplace(group_id, group_task); } // Ungrouped tasks @@ -229,7 +224,7 @@ void CustomGenerator::GenerateTask() { bool build = selected_ids.count(id) == 1; auto task = CreateTaskRunner(subflow, build, id); task.name(id); - registered_tasks.emplace(id, task); + registered_tasks.try_emplace(id, task); } // Dependencies between tasks @@ -283,15 +278,13 @@ void CustomGenerator::TaskRunner(bool run, const std::string &id) { env::log_critical(__FUNCTION__, id); // Convert - { - auto ¤t_gen_info = user_.gen_info_map.at(id); - current_gen_info.internal_inputs = internal::path_schema_convert( - current_gen_info.inputs, internal::Path::CreateExistingPath); - current_gen_info.userblob = - current_gen_info.blob_handler != nullptr - ? current_gen_info.blob_handler->GetSerializedData() - : std::vector(); - } + auto ¤t_gen_info = user_.gen_info_map.at(id); + current_gen_info.internal_inputs = internal::path_schema_convert( + current_gen_info.inputs, internal::Path::CreateExistingPath); + current_gen_info.userblob = + current_gen_info.blob_handler != nullptr + ? current_gen_info.blob_handler->GetSerializedData() + : std::vector(); // Run const auto ¤t_info = user_.gen_info_map.at(id); @@ -319,8 +312,8 @@ void CustomGenerator::TaskRunner(bool run, const std::string &id) { env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id)); } - std::lock_guard guard(success_schema_mutex_); - success_schema_.emplace(id, current_info); + std::scoped_lock guard(success_schema_mutex_); + success_schema_.try_emplace(id, current_info); } } // namespace buildcc From 8da24b9e482c338d5e783d912cef26e14f9df168 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:04:22 -0700 Subject: [PATCH 08/18] Updated custom_generator.h --- .../target/include/target/custom_generator.h | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 155b229b..1f8f9cf9 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -44,20 +44,22 @@ class CustomGeneratorContext { const fs_unordered_set &o) : command(c), inputs(i), outputs(o) {} -public: const env::Command &command; const fs_unordered_set &inputs; const fs_unordered_set &outputs; }; // clang-format off -typedef std::function GenerateCb; +using GenerateCb = std::function; -typedef std::function &&)> DependencyCb; +using DependencyCb = std::function &&)>; // clang-format on class CustomBlobHandler { public: + CustomBlobHandler() = default; + virtual ~CustomBlobHandler() = default; + bool CheckChanged(const std::vector &previous, const std::vector ¤t) const { env::assert_fatal( @@ -69,7 +71,7 @@ class CustomBlobHandler { return !IsEqual(previous, current); }; - std::vector GetSerializedData() { + std::vector GetSerializedData() const { auto serialized_data = Serialize(); env::assert_fatal( Verify(serialized_data), @@ -94,12 +96,11 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { std::unordered_map gen_info_map; void ConvertToInternal() { - for (auto &r_miter : gen_info_map) { - r_miter.second.internal_inputs = path_schema_convert( - r_miter.second.inputs, internal::Path::CreateExistingPath); - auto p = internal_gen_info_map.emplace(r_miter.first, r_miter.second); - env::assert_fatal(p.second, - fmt::format("Could not save {}", r_miter.first)); + for (auto &[id, gen_info] : gen_info_map) { + gen_info.internal_inputs = path_schema_convert( + gen_info.inputs, internal::Path::CreateExistingPath); + auto p = internal_gen_info_map.emplace(id, gen_info); + env::assert_fatal(p.second, fmt::format("Could not save {}", id)); } } }; From 122b99e6ffd5b8c81a73a5b4c90fd968c9066c6a Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:10:15 -0700 Subject: [PATCH 09/18] Updated custom_generator group_metadata implementation --- .../target/include/target/custom_generator.h | 17 +++++++++++++++++ .../src/custom_generator/custom_generator.cpp | 12 +----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 1f8f9cf9..6bbf7323 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -193,6 +193,23 @@ class CustomGenerator : public internal::BuilderInterface { struct GroupMetadata { std::vector ids; DependencyCb dependency_cb; + + void InvokeDependencyCb(const std::string &group_id, + std::unordered_map + &®istered_tasks) const noexcept { + if (!dependency_cb) { + return; + } + try { + dependency_cb(std::move(registered_tasks)); + } catch (...) { + env::log_critical( + __FUNCTION__, + fmt::format("Dependency callback failed for group id {}", + group_id)); + env::set_task_state(env::TaskState::FAILURE); + } + } }; private: diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 6003d70b..5a90274e 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -200,17 +200,7 @@ void CustomGenerator::GenerateTask() { } // Dependency callback - if (group_metadata.dependency_cb) { - try { - group_metadata.dependency_cb(std::move(reg_tasks)); - } catch (...) { - env::log_critical( - __FUNCTION__, - fmt::format("Dependency callback failed for group id {}", - group_id)); - env::set_task_state(env::TaskState::FAILURE); - } - } + group_metadata.InvokeDependencyCb(group_id, std::move(reg_tasks)); // NOTE, Do not call detach otherwise this will fail s.join(); From ce3306ab8a6a8ee420b3ef471664aead579e0fde Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:13:43 -0700 Subject: [PATCH 10/18] Update custom_generator.h --- buildcc/lib/target/include/target/custom_generator.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 6bbf7323..34f0ed5e 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -99,8 +99,8 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { for (auto &[id, gen_info] : gen_info_map) { gen_info.internal_inputs = path_schema_convert( gen_info.inputs, internal::Path::CreateExistingPath); - auto p = internal_gen_info_map.emplace(id, gen_info); - env::assert_fatal(p.second, fmt::format("Could not save {}", id)); + auto [_, success] = internal_gen_info_map.try_emplace(id, gen_info); + env::assert_fatal(success, fmt::format("Could not save {}", id)); } } }; From 70bbc24db7568ed3862b719bc573906c5115ad25 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:20:30 -0700 Subject: [PATCH 11/18] Updated custom_generator --- .../target/include/target/custom_generator.h | 3 +++ .../src/custom_generator/custom_generator.cpp | 24 +++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 34f0ed5e..f79eba5b 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -181,6 +181,9 @@ class CustomGenerator : public internal::BuilderInterface { void BuildGenerate(std::unordered_set &gen_selected_ids, std::unordered_set &dummy_gen_selected_ids); + void InvokeDependencyCb(std::unordered_map + &®istered_tasks) const noexcept; + // Recheck states void IdRemoved(); void IdAdded(); diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 5a90274e..bdaf300c 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -218,14 +218,7 @@ void CustomGenerator::GenerateTask() { } // Dependencies between tasks - if (dependency_cb_) { - try { - dependency_cb_(std::move(registered_tasks)); - } catch (...) { - env::log_critical(__FUNCTION__, "Dependency callback failed"); - env::set_task_state(env::TaskState::FAILURE); - } - } + InvokeDependencyCb(std::move(registered_tasks)); // NOTE, Do not call detach otherwise this will fail subflow.join(); @@ -250,6 +243,19 @@ void CustomGenerator::GenerateTask() { generate_task.name(kGenerateTaskName); } +void CustomGenerator::InvokeDependencyCb( + std::unordered_map &®istered_tasks) + const noexcept { + if (dependency_cb_) { + try { + dependency_cb_(std::move(registered_tasks)); + } catch (...) { + env::log_critical(__FUNCTION__, "Dependency callback failed"); + env::set_task_state(env::TaskState::FAILURE); + } + } +} + tf::Task CustomGenerator::CreateTaskRunner(tf::Subflow &subflow, bool build, const std::string &id) { return subflow.emplace([&, build, id]() { @@ -265,8 +271,6 @@ tf::Task CustomGenerator::CreateTaskRunner(tf::Subflow &subflow, bool build, } void CustomGenerator::TaskRunner(bool run, const std::string &id) { - env::log_critical(__FUNCTION__, id); - // Convert auto ¤t_gen_info = user_.gen_info_map.at(id); current_gen_info.internal_inputs = internal::path_schema_convert( From 0e5936727001648d40cea115dc0821695ace5096 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:36:19 -0700 Subject: [PATCH 12/18] Updated custom_generator --- buildcc/lib/target/include/target/custom_generator.h | 6 ++++-- .../lib/target/src/custom_generator/custom_generator.cpp | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index f79eba5b..e23837df 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -41,12 +41,14 @@ namespace buildcc { class CustomGeneratorContext { public: CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, - const fs_unordered_set &o) - : command(c), inputs(i), outputs(o) {} + const fs_unordered_set &o, + const std::vector &ub) + : command(c), inputs(i), outputs(o), userblob(ub) {} const env::Command &command; const fs_unordered_set &inputs; const fs_unordered_set &outputs; + const std::vector &userblob; }; // clang-format off diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index bdaf300c..c488a54b 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -301,7 +301,8 @@ void CustomGenerator::TaskRunner(bool run, const std::string &id) { if (rerun) { dirty_ = true; buildcc::CustomGeneratorContext ctx(command_, current_info.inputs, - current_info.outputs); + current_info.outputs, + current_info.userblob); bool success = current_info.generate_cb(ctx); env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id)); } From 6553c0e44910c75007afd9a77f0658f5b8af26ae Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:36:29 -0700 Subject: [PATCH 13/18] Updated file generator --- .../target/src/generator/file_generator.cpp | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/buildcc/lib/target/src/generator/file_generator.cpp b/buildcc/lib/target/src/generator/file_generator.cpp index ca94d9e2..ae63956e 100644 --- a/buildcc/lib/target/src/generator/file_generator.cpp +++ b/buildcc/lib/target/src/generator/file_generator.cpp @@ -24,11 +24,22 @@ namespace { -class GeneratorBlobHandler : public buildcc::CustomBlobHandler { +class FileGeneratorBlobHandler : public buildcc::CustomBlobHandler { public: - explicit GeneratorBlobHandler(const std::vector &commands) + explicit FileGeneratorBlobHandler(const std::vector &commands) : commands_(commands) {} + // serialized_data has already been verified + static std::vector + Deserialize(const std::vector &serialized_data) { + std::vector deserialized; + auto flex_vect = flexbuffers::GetRoot(serialized_data).AsVector(); + for (size_t i = 0; i < flex_vect.size(); i++) { + deserialized.emplace_back(flex_vect[i].AsString().str()); + } + return deserialized; + } + private: const std::vector &commands_; @@ -61,18 +72,24 @@ class GeneratorBlobHandler : public buildcc::CustomBlobHandler { builder.Finish(); return builder.GetBuffer(); } +}; - // serialized_data has already been verified - static std::vector - Deserialize(const std::vector &serialized_data) { - std::vector deserialized; - auto flex_vect = flexbuffers::GetRoot(serialized_data).AsVector(); - for (size_t i = 0; i < flex_vect.size(); i++) { - deserialized.emplace_back(flex_vect[i].AsString().str()); +bool FileGeneratorGenerateCb(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + bool success = true; + std::vector commands = + FileGeneratorBlobHandler::Deserialize(ctx.userblob); + for (const auto &c : commands) { + bool executed = buildcc::env::Command::Execute(c); + if (!executed) { + success = false; + buildcc::env::log_critical(__FUNCTION__, + fmt::format("Failed to run command {}", c)); + break; } - return deserialized; } -}; + return success; +} } // namespace @@ -113,23 +130,8 @@ void FileGenerator::AddCommand( } void FileGenerator::Build() { - AddGenInfo( - "Generate", inputs_, outputs_, - [&](const CustomGeneratorContext &ctx) -> bool { - (void)ctx; - bool success = true; - for (const auto &c : commands_) { - bool executed = env::Command::Execute(c); - if (!executed) { - success = false; - env::log_critical(__FUNCTION__, - fmt::format("Failed to run command {}", c)); - break; - } - } - return success; - }, - std::make_shared(commands_)); + AddGenInfo("Generate", inputs_, outputs_, FileGeneratorGenerateCb, + std::make_shared(commands_)); this->CustomGenerator::Build(); } From 7f0d30e60135dd40ffd96d2041a063673bf70c64 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:39:27 -0700 Subject: [PATCH 14/18] Updated template generator --- buildcc/lib/target/include/target/template_generator.h | 2 +- buildcc/lib/target/src/generator/template_generator.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/include/target/template_generator.h b/buildcc/lib/target/include/target/template_generator.h index 17898183..886660fa 100644 --- a/buildcc/lib/target/include/target/template_generator.h +++ b/buildcc/lib/target/include/target/template_generator.h @@ -28,7 +28,7 @@ class TemplateGenerator : public CustomGenerator { void AddTemplate(const fs::path &input_filename, const fs::path &output_filename); - std::string Parse(const std::string &pattern); + std::string Parse(const std::string &pattern) const; /** * @brief Build FileGenerator Tasks diff --git a/buildcc/lib/target/src/generator/template_generator.cpp b/buildcc/lib/target/src/generator/template_generator.cpp index c99fc541..bf10452c 100644 --- a/buildcc/lib/target/src/generator/template_generator.cpp +++ b/buildcc/lib/target/src/generator/template_generator.cpp @@ -56,7 +56,7 @@ void TemplateGenerator::AddTemplate(const fs::path &input_filename, template_infos_.emplace_back(std::move(info)); } -std::string TemplateGenerator::Parse(const std::string &pattern) { +std::string TemplateGenerator::Parse(const std::string &pattern) const { return command_.Construct(pattern); } From c84dcf722087e9ca9fefeb83e55e48b3c46eda84 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:41:26 -0700 Subject: [PATCH 15/18] Updated file_generator --- buildcc/lib/target/include/target/file_generator.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/include/target/file_generator.h b/buildcc/lib/target/include/target/file_generator.h index 27ff1946..cc4b66b0 100644 --- a/buildcc/lib/target/include/target/file_generator.h +++ b/buildcc/lib/target/include/target/file_generator.h @@ -23,9 +23,8 @@ namespace buildcc { class FileGenerator : public CustomGenerator { public: - FileGenerator(const std::string &name, const TargetEnv &env) - : CustomGenerator(name, env) {} - virtual ~FileGenerator() = default; + using CustomGenerator::CustomGenerator; + ~FileGenerator() override = default; FileGenerator(const FileGenerator &) = delete; /** From 565a7765aef36c147c9ca44ccf4233e8353c23fd Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:42:44 -0700 Subject: [PATCH 16/18] Updated template_generator --- buildcc/lib/target/include/target/template_generator.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/include/target/template_generator.h b/buildcc/lib/target/include/target/template_generator.h index 886660fa..b22d49c0 100644 --- a/buildcc/lib/target/include/target/template_generator.h +++ b/buildcc/lib/target/include/target/template_generator.h @@ -23,8 +23,9 @@ namespace buildcc { class TemplateGenerator : public CustomGenerator { public: - TemplateGenerator(const std::string &name, const TargetEnv &env) - : CustomGenerator(name, env) {} + using CustomGenerator::CustomGenerator; + ~TemplateGenerator() override = default; + TemplateGenerator(const TemplateGenerator &) = delete; void AddTemplate(const fs::path &input_filename, const fs::path &output_filename); From d6cc12a18b960916a33171412fd019c018e97734 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 16 May 2022 22:47:53 -0700 Subject: [PATCH 17/18] Updated file and template generator --- buildcc/lib/target/include/target/custom_generator.h | 4 +++- buildcc/lib/target/src/generator/file_generator.cpp | 6 +++--- buildcc/lib/target/src/generator/template_generator.cpp | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index e23837df..316290ea 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -192,7 +192,8 @@ class CustomGenerator : public internal::BuilderInterface { void IdUpdated(); protected: - env::Command command_; + const env::Command &ConstCommand() const { return command_; } + env::Command &RefCommand() { return command_; } private: struct GroupMetadata { @@ -231,6 +232,7 @@ class CustomGenerator : public internal::BuilderInterface { std::unordered_map success_schema_; // Internal + env::Command command_; tf::Taskflow tf_; // Callbacks diff --git a/buildcc/lib/target/src/generator/file_generator.cpp b/buildcc/lib/target/src/generator/file_generator.cpp index ae63956e..1362ed32 100644 --- a/buildcc/lib/target/src/generator/file_generator.cpp +++ b/buildcc/lib/target/src/generator/file_generator.cpp @@ -98,7 +98,7 @@ namespace buildcc { void FileGenerator::AddInput(const std::string &absolute_input_pattern, const char *identifier) { std::string absolute_input_string = - command_.Construct(absolute_input_pattern); + RefCommand().Construct(absolute_input_pattern); const auto absolute_input_path = internal::Path::CreateNewPath(absolute_input_string); inputs_.insert(absolute_input_path.GetPathname()); @@ -111,7 +111,7 @@ void FileGenerator::AddInput(const std::string &absolute_input_pattern, void FileGenerator::AddOutput(const std::string &absolute_output_pattern, const char *identifier) { std::string absolute_output_string = - command_.Construct(absolute_output_pattern); + RefCommand().Construct(absolute_output_pattern); const auto absolute_output_path = internal::Path::CreateNewPath(absolute_output_string); outputs_.insert(absolute_output_path.GetPathname()); @@ -125,7 +125,7 @@ void FileGenerator::AddCommand( const std::string &command_pattern, const std::unordered_map &arguments) { std::string constructed_command = - command_.Construct(command_pattern, arguments); + RefCommand().Construct(command_pattern, arguments); commands_.emplace_back(std::move(constructed_command)); } diff --git a/buildcc/lib/target/src/generator/template_generator.cpp b/buildcc/lib/target/src/generator/template_generator.cpp index bf10452c..e7c059c0 100644 --- a/buildcc/lib/target/src/generator/template_generator.cpp +++ b/buildcc/lib/target/src/generator/template_generator.cpp @@ -48,16 +48,16 @@ void TemplateGenerator::AddTemplate(const fs::path &input_filename, const fs::path &output_filename) { TemplateInfo info; info.input = internal::Path::CreateNewPath( - command_.Construct(path_as_string(input_filename))) + RefCommand().Construct(path_as_string(input_filename))) .GetPathname(); info.output = internal::Path::CreateNewPath( - command_.Construct(path_as_string(output_filename))) + RefCommand().Construct(path_as_string(output_filename))) .GetPathname(); template_infos_.emplace_back(std::move(info)); } std::string TemplateGenerator::Parse(const std::string &pattern) const { - return command_.Construct(pattern); + return ConstCommand().Construct(pattern); } /** From 201945ec8a3c86fa0cc45e11d58b8058f4f70bae Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Tue, 17 May 2022 20:24:29 -0700 Subject: [PATCH 18/18] Update test_custom_generator.cpp --- buildcc/lib/target/test/target/test_custom_generator.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 44dc65cf..ea3eef74 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -746,6 +746,9 @@ TEST(CustomGeneratorTestGroup, RealGenerate_RemoveAndAdd) { {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddDependencyCb([](auto &&task_map) { + task_map.at("id1").precede(task_map.at("id2")); + }); cgen.Build(); buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); @@ -807,6 +810,9 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Failure) { {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.cpp"}, {"{gen_build_dir}/other_dummy_main.o"}, RealGenerateCb); + cgen.AddDependencyCb([](auto &&task_map) { + task_map.at("id1").precede(task_map.at("id2")); + }); cgen.Build(); mock().expectOneCall("RealGenerateCb"); @@ -840,6 +846,9 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Failure) { {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.cpp"}, {"{gen_build_dir}/other_dummy_main.o"}, RealGenerateCb); + cgen.AddDependencyCb([](auto &&task_map) { + task_map.at("id1").precede(task_map.at("id2")); + }); cgen.Build(); mock().expectOneCall("RealGenerateCb");