diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 0712cde5..a9dec033 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -39,29 +39,25 @@ namespace buildcc { struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { struct UserIdInfo : internal::CustomGeneratorSchema::IdInfo { - fs_unordered_set inputs; // TODO, Remove - GenerateCb generate_cb; - std::shared_ptr blob_handler{nullptr}; - void ConvertToInternal() { - internal_inputs = internal::path_schema_convert( - inputs, internal::Path::CreateExistingPath); + inputs.ComputeHashForAll(); userblob = blob_handler != nullptr ? blob_handler->GetSerializedData() : std::vector(); } - }; - using UserIdPair = std::pair; - std::unordered_map ids; + GenerateCb generate_cb; + std::shared_ptr blob_handler{nullptr}; + }; void ConvertToInternal() { for (auto &[id_key, id_info] : ids) { - id_info.internal_inputs = path_schema_convert( - id_info.inputs, internal::Path::CreateExistingPath); + id_info.ConvertToInternal(); auto [_, success] = internal_ids.try_emplace(id_key, id_info); env::assert_fatal(success, fmt::format("Could not save {}", id_key)); } } + + std::unordered_map ids; }; class CustomGenerator : public internal::BuilderInterface { @@ -69,8 +65,8 @@ class CustomGenerator : public internal::BuilderInterface { CustomGenerator(const std::string &name, const TargetEnv &env) : name_(name), env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), - serialization_(env_.GetTargetBuildDir() / fmt::format("{}.json", name)), - comparator_(serialization_.GetLoad(), user_) { + serialization_(env_.GetTargetBuildDir() / + fmt::format("{}.json", name)) { Initialize(); } virtual ~CustomGenerator() = default; @@ -115,100 +111,9 @@ class CustomGenerator : public internal::BuilderInterface { const fs::path &GetBuildDir() const { return env_.GetTargetBuildDir(); } const std::string &Get(const std::string &file_identifier) const; -private: - struct Comparator { - Comparator(const internal::CustomGeneratorSchema &loaded, - const UserCustomGeneratorSchema &us) - : loaded_schema_(loaded), current_schema_(us) {} - - enum class State { - kRemoved, - kAdded, - kCheckLater, - }; - - void AddAllIds() { - const auto &curr_ids = current_schema_.ids; - for (const auto &[id, _] : curr_ids) { - id_state_info_.at(State::kAdded).insert(id); - } - } - - void CompareIds() { - const auto &prev_ids = loaded_schema_.internal_ids; - const auto &curr_ids = current_schema_.ids; - - for (const auto &[prev_id, _] : prev_ids) { - if (curr_ids.find(prev_id) == curr_ids.end()) { - // Id Removed condition, previous id is not present in the current run - id_state_info_.at(State::kRemoved).insert(prev_id); - } - } - - for (const auto &[curr_id, _] : curr_ids) { - if (prev_ids.find(curr_id) == prev_ids.end()) { - // Id Added condition - id_state_info_.at(State::kAdded).insert(curr_id); - } else { - // Id Check Later condition - id_state_info_.at(State::kCheckLater).insert(curr_id); - } - } - } - - bool IsChanged(const std::string &id) const { - const auto &previous_id_info = loaded_schema_.internal_ids.at(id); - const auto ¤t_id_info = current_schema_.ids.at(id); - - bool changed = internal::CheckPaths(previous_id_info.internal_inputs, - current_id_info.internal_inputs) != - internal::PathState::kNoChange; - changed = changed || internal::CheckChanged(previous_id_info.outputs, - current_id_info.outputs); - if (!changed && current_id_info.blob_handler != nullptr) { - // We only check blob handler if not changed by inputs/outputs - // Checking blob_handler could be expensive so this optimization is made - // to run only when changed == false - changed = current_id_info.blob_handler->CheckChanged( - previous_id_info.userblob, current_id_info.userblob); - } - return changed; - } - - const std::unordered_set &GetRemovedIds() const { - return id_state_info_.at(State::kRemoved); - } - - const std::unordered_set &GetAddedIds() const { - return id_state_info_.at(State::kAdded); - } - - const std::unordered_set &GetCheckLaterIds() const { - return id_state_info_.at(State::kCheckLater); - } - - bool IsIdAdded(const std::string &id) const { - return id_state_info_.at(State::kAdded).count(id) == 1; - } - - private: - const internal::CustomGeneratorSchema &loaded_schema_; - const UserCustomGeneratorSchema ¤t_schema_; - std::unordered_map> id_state_info_{ - {State::kRemoved, std::unordered_set()}, - {State::kAdded, std::unordered_set()}, - {State::kCheckLater, std::unordered_set()}, - }; - }; - private: void Initialize(); - - tf::Task CreateTaskRunner(tf::Subflow &subflow, const std::string &id); - void TaskRunner(const std::string &id); - void GenerateTask(); - void BuildGenerate(); // Recheck states void IdRemoved(); @@ -227,13 +132,6 @@ class CustomGenerator : public internal::BuilderInterface { // Serialization UserCustomGeneratorSchema user_; - // Comparator - Comparator comparator_; - - std::mutex success_schema_mutex_; - std::unordered_map - success_schema_; - // Internal env::Command command_; }; diff --git a/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h b/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h index 1c268301..3b64d23c 100644 --- a/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h +++ b/buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h @@ -23,6 +23,12 @@ namespace buildcc { +/** + * @brief Abstract class for serializing additional data for which rebuilds + * might be triggered i.e data that is not input/output files + * TODO, Add examples here + * + */ class CustomBlobHandler { public: CustomBlobHandler() = default; @@ -54,6 +60,48 @@ class CustomBlobHandler { virtual std::vector Serialize() const = 0; }; +/** + * @brief Typed Custom Blob handler which automatically performs Serialization + * and Deserialization as long as it is JSON serializable + * + * NOTE: Type data is stored as a reference (to avoid copying large amount of + * data) when constructing TypedCustomBlobHandler + * + * @tparam Type should be JSON serializable (see nlohmann::json compatible + * objects) + */ +template +class TypedCustomBlobHandler : public CustomBlobHandler { +public: + explicit TypedCustomBlobHandler(const Type &data) : data_(data) {} + + // serialized_data has already been verified + static Type Deserialize(const std::vector &serialized_data) { + json j = json::from_msgpack(serialized_data, true, false); + Type deserialized; + j.get_to(deserialized); + return deserialized; + } + +private: + const Type &data_; + + bool Verify(const std::vector &serialized_data) const override { + json j = json::from_msgpack(serialized_data, true, false); + return !j.is_discarded(); + } + + bool IsEqual(const std::vector &previous, + const std::vector ¤t) const override { + return Deserialize(previous) == Deserialize(current); + } + + std::vector Serialize() const override { + json j = data_; + return json::to_msgpack(j); + } +}; + } // namespace buildcc #endif diff --git a/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h b/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h index 207015d9..4c49d5ff 100644 --- a/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h +++ b/buildcc/lib/target/include/target/custom_generator/custom_generator_context.h @@ -25,19 +25,20 @@ namespace buildcc { class CustomGeneratorContext { public: - CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, - const fs_unordered_set &o, + CustomGeneratorContext(const env::Command &c, + const std::unordered_set &i, + const std::unordered_set &o, const std::vector &ub) : command(c), inputs(i), outputs(o), userblob(ub) {} const env::Command &command; - const fs_unordered_set &inputs; - const fs_unordered_set &outputs; + const std::unordered_set &inputs; + const std::unordered_set &outputs; const std::vector &userblob; }; // clang-format off -using GenerateCb = std::function; +using GenerateCb = std::function; // clang-format on } // namespace buildcc diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index b60283e8..138bb9f3 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -28,6 +28,164 @@ constexpr const char *const kCurrentBuildDirName = "current_build_dir"; namespace buildcc { +struct Comparator { + Comparator(const internal::CustomGeneratorSchema &loaded, + const UserCustomGeneratorSchema ¤t) + : loaded_(loaded), current_(current) {} + + enum class State { + kRemoved, + kAdded, + kCheckLater, + }; + + void AddAllIds() { + const auto &curr_ids = current_.ids; + for (const auto &[id, _] : curr_ids) { + id_state_info_.at(State::kAdded).insert(id); + } + } + + void CompareAndAddIds() { + const auto &prev_ids = loaded_.internal_ids; + const auto &curr_ids = current_.ids; + + for (const auto &[prev_id, _] : prev_ids) { + if (curr_ids.find(prev_id) == curr_ids.end()) { + // Id Removed condition, previous id is not present in the current run + id_state_info_.at(State::kRemoved).insert(prev_id); + } + } + + for (const auto &[curr_id, _] : curr_ids) { + if (prev_ids.find(curr_id) == prev_ids.end()) { + // Id Added condition + id_state_info_.at(State::kAdded).insert(curr_id); + } else { + // Id Check Later condition + id_state_info_.at(State::kCheckLater).insert(curr_id); + } + } + } + + bool IsChanged(const std::string &id) const { + const auto &previous_id_info = loaded_.internal_ids.at(id); + const auto ¤t_id_info = current_.ids.at(id); + + bool changed = !previous_id_info.inputs.IsEqual(current_id_info.inputs) || + !previous_id_info.outputs.IsEqual(current_id_info.outputs); + if (!changed && current_id_info.blob_handler != nullptr) { + // We only check blob handler if not changed by inputs/outputs + // Checking blob_handler could be expensive so this optimization is made + // to run only when changed == false + changed = current_id_info.blob_handler->CheckChanged( + previous_id_info.userblob, current_id_info.userblob); + } + return changed; + } + + const std::unordered_set &GetRemovedIds() const { + return id_state_info_.at(State::kRemoved); + } + + const std::unordered_set &GetAddedIds() const { + return id_state_info_.at(State::kAdded); + } + + const std::unordered_set &GetCheckLaterIds() const { + return id_state_info_.at(State::kCheckLater); + } + + bool IsIdAdded(const std::string &id) const { + return id_state_info_.at(State::kAdded).count(id) == 1; + } + +private: + const buildcc::internal::CustomGeneratorSchema &loaded_; + const buildcc::UserCustomGeneratorSchema ¤t_; + std::unordered_map> id_state_info_{ + {State::kRemoved, std::unordered_set()}, + {State::kAdded, std::unordered_set()}, + {State::kCheckLater, std::unordered_set()}, + }; +}; + +struct TaskState { + bool should_run{false}; + bool run_success{false}; +}; + +struct TaskFunctor { + TaskFunctor(const std::string &id, + UserCustomGeneratorSchema::UserIdInfo &id_info, + const Comparator &comparator, const env::Command &command, + TaskState &state) + : id_(id), id_info_(id_info), comparator(comparator), command_(command), + state_(state) {} + + void operator()() { + if (env::get_task_state() != env::TaskState::SUCCESS) { + return; + } + try { + id_info_.ConvertToInternal(); + // Compute runnable + state_.should_run = + comparator.IsIdAdded(id_) ? true : comparator.IsChanged(id_); + + // Invoke generator callback + if (state_.should_run) { + const auto input_paths = id_info_.inputs.GetPaths(); + CustomGeneratorContext ctx(command_, input_paths, + id_info_.outputs.GetPaths(), + id_info_.userblob); + + bool success = id_info_.generate_cb(ctx); + env::assert_fatal(success, + fmt::format("Generate Cb failed for id {}", id_)); + } + state_.run_success = true; + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + } + +private: + const std::string &id_; + UserCustomGeneratorSchema::UserIdInfo &id_info_; + + const Comparator &comparator; + const env::Command &command_; + + TaskState &state_; +}; + +bool ComputeBuild(const internal::CustomGeneratorSerialization &serialization, + Comparator &comparator, + std::function &&id_removed_cb, + std::function &&id_added_cb) { + bool build = false; + if (!serialization.IsLoaded()) { + comparator.AddAllIds(); + build = true; + } else { + comparator.CompareAndAddIds(); + const bool is_removed = !comparator.GetRemovedIds().empty(); + const bool is_added = !comparator.GetAddedIds().empty(); + build = is_removed || is_added; + + if (is_removed) { + id_removed_cb(); + } + + for (const auto &id : comparator.GetAddedIds()) { + (void)id; + id_added_cb(); + } + } + return build; +} + void CustomGenerator::AddPattern(const std::string &identifier, const std::string &pattern) { command_.AddDefaultArgument(identifier, command_.Construct(pattern)); @@ -62,12 +220,12 @@ void CustomGenerator::AddIdInfo( UserCustomGeneratorSchema::UserIdInfo schema; for (const auto &i : inputs) { - fs::path input = string_as_path(command_.Construct(i)); - schema.inputs.emplace(std::move(input)); + auto input = command_.Construct(i); + schema.inputs.Emplace(input, ""); } for (const auto &o : outputs) { - fs::path output = string_as_path(command_.Construct(o)); - schema.outputs.emplace(std::move(output)); + auto output = command_.Construct(o); + schema.outputs.Emplace(output); } schema.generate_cb = generate_cb; schema.blob_handler = blob_handler; @@ -76,7 +234,6 @@ void CustomGenerator::AddIdInfo( void CustomGenerator::Build() { (void)serialization_.LoadFromFile(); - GenerateTask(); } @@ -101,29 +258,6 @@ void CustomGenerator::Initialize() { tf_.name(name_); } -void CustomGenerator::BuildGenerate() { - if (!serialization_.IsLoaded()) { - comparator_.AddAllIds(); - dirty_ = true; - } else { - // For IDS - comparator_.CompareIds(); - - const bool is_removed = !comparator_.GetRemovedIds().empty(); - const bool is_added = !comparator_.GetAddedIds().empty(); - dirty_ = is_removed || is_added; - - if (is_removed) { - IdRemoved(); - } - - for (const auto &id : comparator_.GetAddedIds()) { - (void)id; - IdAdded(); - } - } -} - void CustomGenerator::GenerateTask() { tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { if (env::get_task_state() != env::TaskState::SUCCESS) { @@ -132,29 +266,43 @@ void CustomGenerator::GenerateTask() { try { // Selected ids for build - BuildGenerate(); + Comparator comparator(serialization_.GetLoad(), user_); + dirty_ = ComputeBuild( + serialization_, comparator, [this]() { IdRemoved(); }, + [this]() { IdAdded(); }); + + std::unordered_map states; // Create runner for each added/updated id - for (const auto &id : comparator_.GetAddedIds()) { - auto task = CreateTaskRunner(subflow, id); - task.name(id); + for (const auto &id : comparator.GetAddedIds()) { + states.try_emplace(id, TaskState()); + auto &id_info = user_.ids.at(id); + TaskFunctor functor(id, id_info, comparator, command_, states.at(id)); + subflow.emplace(functor).name(id); } - for (const auto &id : comparator_.GetCheckLaterIds()) { - auto task = CreateTaskRunner(subflow, id); - task.name(id); + for (const auto &id : comparator.GetCheckLaterIds()) { + states.try_emplace(id, TaskState()); + auto &id_info = user_.ids.at(id); + TaskFunctor functor(id, id_info, comparator, command_, states.at(id)); + subflow.emplace(functor).name(id); } // NOTE, Do not call detach otherwise this will fail subflow.join(); + UserCustomGeneratorSchema user_final_schema; + for (const auto &[id, state] : states) { + dirty_ = dirty_ || state.should_run; + if (state.run_success) { + user_final_schema.ids.try_emplace(id, user_.ids.at(id)); + } + } + // Store if (dirty_) { - UserCustomGeneratorSchema user_final_schema; - user_final_schema.ids.insert(success_schema_.begin(), - success_schema_.end()); - user_final_schema.ConvertToInternal(); + serialization_.UpdateStore(user_final_schema); env::assert_fatal(serialization_.StoreToFile(), fmt::format("Store failed for {}", name_)); @@ -163,44 +311,7 @@ void CustomGenerator::GenerateTask() { env::set_task_state(env::TaskState::FAILURE); } }); - // TODO, Instead of "Generate" name the task of user's choice generate_task.name(kGenerateTaskName); } -tf::Task CustomGenerator::CreateTaskRunner(tf::Subflow &subflow, - const std::string &id) { - return subflow.emplace([&, id]() { - if (env::get_task_state() != env::TaskState::SUCCESS) { - return; - } - try { - TaskRunner(id); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }); -} - -void CustomGenerator::TaskRunner(const std::string &id) { - // Convert to internal - user_.ids.at(id).ConvertToInternal(); - - // Compute runnable - bool run = comparator_.IsIdAdded(id) ? true : comparator_.IsChanged(id); - - // Invoke generator callback - const auto ¤t_id_info = user_.ids.at(id); - if (run) { - dirty_ = true; - CustomGeneratorContext ctx(command_, current_id_info.inputs, - current_id_info.outputs, - current_id_info.userblob); - bool success = current_id_info.generate_cb(ctx); - env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id)); - } - - std::scoped_lock guard(success_schema_mutex_); - success_schema_.try_emplace(id, current_id_info); -} - } // namespace buildcc diff --git a/buildcc/lib/target/src/generator/file_generator.cpp b/buildcc/lib/target/src/generator/file_generator.cpp index 5cdb592f..3256ad2b 100644 --- a/buildcc/lib/target/src/generator/file_generator.cpp +++ b/buildcc/lib/target/src/generator/file_generator.cpp @@ -22,44 +22,12 @@ namespace { -class FileGeneratorBlobHandler : public buildcc::CustomBlobHandler { -public: - explicit FileGeneratorBlobHandler(const std::vector &commands) - : commands_(commands) {} - - // serialized_data has already been verified - static std::vector - Deserialize(const std::vector &serialized_data) { - json j = json::from_msgpack(serialized_data, true, false); - std::vector deserialized; - j.get_to(deserialized); - return deserialized; - } - -private: - const std::vector &commands_; - - bool Verify(const std::vector &serialized_data) const override { - json j = json::from_msgpack(serialized_data, true, false); - return !j.is_discarded(); - } - - bool IsEqual(const std::vector &previous, - const std::vector ¤t) const override { - return Deserialize(previous) == Deserialize(current); - } - - std::vector Serialize() const override { - json j = commands_; - return json::to_msgpack(j); - } -}; - bool FileGeneratorGenerateCb(const buildcc::CustomGeneratorContext &ctx) { (void)ctx; bool success = true; std::vector commands = - FileGeneratorBlobHandler::Deserialize(ctx.userblob); + buildcc::TypedCustomBlobHandler>::Deserialize( + ctx.userblob); for (const auto &c : commands) { bool executed = buildcc::env::Command::Execute(c); if (!executed) { @@ -92,8 +60,11 @@ void FileGenerator::AddCommand( } void FileGenerator::Build() { + auto file_blob_handler = + std::make_shared>>( + commands_); AddIdInfo("Generate", inputs_, outputs_, FileGeneratorGenerateCb, - std::make_shared(commands_)); + file_blob_handler); this->CustomGenerator::Build(); } diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 839c91f7..f4293c9f 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -54,12 +54,12 @@ TEST(CustomGeneratorTestGroup, Basic) { const auto &internal_map = serialization.GetLoad().internal_ids; CHECK_EQUAL(internal_map.size(), 2); const auto &id1_info = internal_map.at("id1"); - CHECK_EQUAL(id1_info.internal_inputs.size(), 1); - CHECK_EQUAL(id1_info.outputs.size(), 1); + CHECK_EQUAL(id1_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id1_info.outputs.GetPaths().size(), 1); const auto &id2_info = internal_map.at("id2"); - CHECK_EQUAL(id2_info.internal_inputs.size(), 1); - CHECK_EQUAL(id2_info.outputs.size(), 0); + CHECK_EQUAL(id2_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id2_info.outputs.GetPaths().size(), 0); } } @@ -184,12 +184,12 @@ TEST(CustomGeneratorTestGroup, DefaultArgumentUsage) { const auto &internal_map = serialization.GetLoad().internal_ids; CHECK_EQUAL(internal_map.size(), 2); const auto &id1_info = internal_map.at("id1"); - CHECK_EQUAL(id1_info.internal_inputs.size(), 1); - CHECK_EQUAL(id1_info.outputs.size(), 1); + CHECK_EQUAL(id1_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id1_info.outputs.GetPaths().size(), 1); const auto &id2_info = internal_map.at("id2"); - CHECK_EQUAL(id2_info.internal_inputs.size(), 1); - CHECK_EQUAL(id2_info.outputs.size(), 0); + CHECK_EQUAL(id2_info.inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(id2_info.outputs.GetPaths().size(), 0); } } @@ -332,11 +332,11 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK_TRUE(serialization.LoadFromFile()); CHECK_EQUAL(serialization.GetLoad().internal_ids.size(), 2); auto imap = serialization.GetLoad().internal_ids; - CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id1").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id2").inputs.GetPathInfos().size(), 1); - CHECK_EQUAL(imap.at("id1").outputs.size(), 1); - CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.GetPaths().size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.GetPaths().size(), 1); } buildcc::m::blocking_sleep(1); @@ -348,7 +348,7 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { buildcc::env::save_file( (cgen.GetBuildDir() / "dummy_main.cpp").string().c_str(), "", false); - std::uint64_t last_write_timestamp = static_cast( + auto last_write_timestamp = static_cast( fs::last_write_time(cgen.GetBuildDir() / "dummy_main.cpp") .time_since_epoch() .count()); @@ -368,14 +368,14 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK_TRUE(serialization.LoadFromFile()); CHECK_EQUAL(serialization.GetLoad().internal_ids.size(), 2); auto imap = serialization.GetLoad().internal_ids; - CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id1").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.GetPaths().size(), 1); - CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.GetPaths().size(), 1); - CHECK_EQUAL(last_write_timestamp, - imap.at("id2").internal_inputs.begin()->last_write_timestamp); + STRCMP_EQUAL(std::to_string(last_write_timestamp).c_str(), + imap.at("id2").inputs.GetPathInfos().begin()->second.c_str()); CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); } @@ -400,11 +400,11 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK_TRUE(serialization.LoadFromFile()); CHECK_EQUAL(serialization.GetLoad().internal_ids.size(), 2); auto imap = serialization.GetLoad().internal_ids; - CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id1").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.GetPaths().size(), 1); - CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); - CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").inputs.GetPathInfos().size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.GetPaths().size(), 1); CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); } diff --git a/buildcc/schema/include/schema/custom_generator_schema.h b/buildcc/schema/include/schema/custom_generator_schema.h index d7027b98..a4088a66 100644 --- a/buildcc/schema/include/schema/custom_generator_schema.h +++ b/buildcc/schema/include/schema/custom_generator_schema.h @@ -37,18 +37,18 @@ struct CustomGeneratorSchema { static constexpr const char *const kUserblob = "userblob"; public: - path_unordered_set internal_inputs; - fs_unordered_set outputs; + PathInfoList inputs; + PathList outputs; std::vector userblob; friend void to_json(json &j, const IdInfo &info) { - j[kInputs] = info.internal_inputs; + j[kInputs] = info.inputs; j[kOutputs] = info.outputs; j[kUserblob] = info.userblob; } friend void from_json(const json &j, IdInfo &info) { - j.at(kInputs).get_to(info.internal_inputs); + j.at(kInputs).get_to(info.inputs); j.at(kOutputs).get_to(info.outputs); j.at(kUserblob).get_to(info.userblob); } diff --git a/buildcc/schema/include/schema/path.h b/buildcc/schema/include/schema/path.h index 755fa6db..9e1107d3 100644 --- a/buildcc/schema/include/schema/path.h +++ b/buildcc/schema/include/schema/path.h @@ -34,6 +34,7 @@ using json = nlohmann::ordered_json; namespace buildcc::internal { +// TODO, Update this struct Path { private: static constexpr const char *const kPathName = "path"; @@ -92,6 +93,43 @@ struct Path { return pathname == other_pathname; } + /** + * @brief Sanitizes a fs::path or std::string to a standard path string + * - Converts backslash (\) to forward slash (/) + * - Makes fs::lexically_normal (see std::filesystem library impl) + * + * @param str User provided fs::path/std::string + * @return std::string Sanitized path as std::string + */ + static std::string ToPathString(const std::string &str) { + auto path_str = str; + std::replace(path_str.begin(), path_str.end(), '\\', '/'); + path_str = fs::path(path_str).lexically_normal().string(); + return path_str; + } + + /** + * @brief Formats a fs::path or std::string for display + * - All the sanitization as done in `ToPathString` + * Additionally + * - Adds quotation marks ("") when a space is detected + * For example: test/hello world/ -> "test/hello world/" + * + * NOTE: Use this API only in places where you would like to output to + * console/run or create command through subprocess + * + * @param str User provided fs::path/std::string + * @return std::string Sanitized path as std::string for display + */ + static std::string ToPathDisplayString(const std::string &str) { + auto path_str = ToPathString(str); + // if spaces are present in the path string, surround this with brackets + if (path_str.find(' ') != std::string::npos) { + path_str = fmt::format("\"{}\"", path_str); + } + return path_str; + } + // JSON specialization friend void to_json(json &j, const Path &p) { @@ -124,6 +162,7 @@ struct Path { std::uint64_t last_write_timestamp{0}; }; +// TODO, Remove this // Used by Path class PathHash { public: @@ -132,9 +171,122 @@ class PathHash { size_t operator()(const fs::path &p) const { return fs::hash_value(p); } }; +// TODO, Remove this using path_unordered_set = std::unordered_set; using fs_unordered_set = std::unordered_set; +/** + * @brief Stores path + */ +class PathList { +public: + PathList() = default; + PathList(std::initializer_list paths) { + for (const auto &path : paths) { + Emplace(path); + } + } + + void Emplace(const std::string &pstr) { + auto path_str = Path::ToPathString(pstr); + paths_.emplace(std::move(path_str)); + } + + bool IsEqual(const PathList &other) const { return paths_ == other.paths_; } + const std::unordered_set &GetPaths() const { return paths_; } + + friend void to_json(json &j, const PathList &plist) { j = plist.paths_; } + + friend void from_json(const json &j, PathList &plist) { + j.get_to(plist.paths_); + } + +private: + std::unordered_set paths_; +}; + +/** + * @brief Stores path + path hash in a hashmap + * + */ +class PathInfoList { +private: + static constexpr const char *const kPath = "path"; + static constexpr const char *const kHash = "hash"; + +public: + PathInfoList() = default; + explicit PathInfoList( + std::initializer_list> + path_infos) { + for (const auto &pinfo : path_infos) { + Emplace(pinfo.first, pinfo.second); + } + } + + void Emplace(const std::string &pstr, const std::string &hash) { + auto path_str = Path::ToPathString(pstr); + path_infos_.emplace(std::move(path_str), hash); + } + + void ComputeHashForAll() { + for (auto &[path_str, hash] : path_infos_) { + hash = ComputeHash(path_str); + } + } + + const std::string &GetHash(const std::string &str) const { + auto path_str = Path::ToPathString(str); + const bool found = path_infos_.find(path_str) != path_infos_.end(); + env::assert_fatal(found, ""); + return path_infos_.at(path_str); + } + + bool IsEqual(const PathInfoList &other) const { + return path_infos_ == other.path_infos_; + } + + const std::unordered_map &GetPathInfos() const { + return path_infos_; + } + + std::unordered_set GetPaths() const { + std::unordered_set paths; + for (const auto &[path_str, hash] : path_infos_) { + paths.emplace(path_str); + } + return paths; + } + + // TODO, Add Compute Strategy enum + static std::string ComputeHash(const std::string &pstr) { + auto path_str = Path::ToPathString(pstr); + + // TODO, There might be a file checksum hash compute strategy + // This is the timestamp hash compute strategy + std::error_code errcode; + const std::uint64_t last_write_timestamp = + std::filesystem::last_write_time(path_str, errcode) + .time_since_epoch() + .count(); + env::assert_fatal(errcode.value() == 0, + fmt::format("{} not found", path_str)); + return std::to_string(last_write_timestamp); + } + + friend void to_json(json &j, const PathInfoList &plist) { + j = plist.path_infos_; + } + + friend void from_json(const json &j, PathInfoList &plist) { + j.get_to(plist.path_infos_); + } + +private: + std::unordered_map path_infos_; +}; + +// TODO, Remove this inline std::vector path_schema_convert(const std::vector &path_list, const std::function &cb = @@ -146,6 +298,7 @@ path_schema_convert(const std::vector &path_list, return internal_path_list; } +// TODO, Remove this inline path_unordered_set path_schema_convert(const fs_unordered_set &path_set, const std::function &cb = @@ -157,6 +310,7 @@ path_schema_convert(const fs_unordered_set &path_set, return internal_path_set; } +// TODO, Remove this inline fs_unordered_set path_schema_convert(const path_unordered_set &internal_path_set) { fs_unordered_set path_set;