From 13cb57a827027a3518801fe58a40c6ffc15ebbd1 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 14:16:45 -0400 Subject: [PATCH 01/15] main cmake changes --- CMakeLists.txt | 27 +++++++++++++++++++++------ external_parser/CMakeLists.txt | 16 +++++++++++++--- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ee51d28b..27db66aad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,13 @@ option(RL_USE_UBSAN "Compile with UndefinedBehaviorSanitizer" OFF) option(rlclientlib_BUILD_ONNXRUNTIME_EXTENSION "Build OnnxRuntime Inference Extension" OFF) option(rlclientlib_BUILD_DOTNET "Build .NET bindings" OFF) option(rlclientlib_DOTNET_USE_MSPROJECT "[Experimental] Use import_external_msproject to build .NET csproj files." OFF) +option(RL_BUILD_FEDERATION "Build code for Federated Learning" OFF) + +if(RL_BUILD_FEDERATION) + add_compile_definitions(RL_BUILD_FEDERATION) + # Needed for joiner code + set(RL_BUILD_EXTERNAL_PARSER ON CACHE BOOL "" FORCE) +endif() if(RL_USE_ASAN) add_compile_definitions(RL_USE_ASAN VW_USE_ASAN) @@ -81,8 +88,9 @@ if(RL_USE_UBSAN) if(MSVC) message(FATAL_ERROR "UBSan not supported on MSVC") else() - add_compile_options(-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer -g3) - add_link_options(-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer -g3) + # Flatbuffers gives errors with misaligned pointer sanitization, so disable it + add_compile_options(-fsanitize=undefined -fno-sanitize=alignment -fno-sanitize-recover -fno-omit-frame-pointer -g3) + add_link_options(-fsanitize=undefined -fno-sanitize=alignment -fno-sanitize-recover -fno-omit-frame-pointer -g3) endif() endif() @@ -140,6 +148,10 @@ include(GNUInstallDirs) include(ext_libs/ext_libs.cmake) +if(RL_BUILD_EXTERNAL_PARSER) + add_subdirectory(external_parser) +endif() + add_subdirectory(rlclientlib) add_subdirectory(rlclientlib/extensions) add_subdirectory(examples) @@ -147,10 +159,6 @@ add_subdirectory(test_tools/joiner) add_subdirectory(test_tools/sender_test) add_subdirectory(test_tools/example_gen) -if(RL_BUILD_EXTERNAL_PARSER) - add_subdirectory(external_parser) -endif() - # enable_testing should be run after ext_libs so that the vw unit tests arent turned on. enable_testing() @@ -168,6 +176,13 @@ if(RL_BUILD_BENCHMARKS) add_subdirectory(benchmarks) endif() +# Add a target to generate all flatbuffer header files, used for clang-tidy +add_custom_target(fbgen) +add_dependencies(fbgen fbgenerator_v1 fbgenerator_v2) +if(TARGET fbgen_external_parser) + add_dependencies(fbgen fbgen_external_parser) +endif() + # Add the nuget subdirectory last if(RL_BUILD_NUGET) if(WIN32) diff --git a/external_parser/CMakeLists.txt b/external_parser/CMakeLists.txt index 99a822c0d..e828a30dc 100644 --- a/external_parser/CMakeLists.txt +++ b/external_parser/CMakeLists.txt @@ -120,7 +120,7 @@ set(RL_FLAT_BUFFER_FILES ) add_flatbuffer_schema( - TARGET fbgen + TARGET fbgen_external_parser SCHEMAS ${RL_FLAT_BUFFER_FILES} OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/generated/v2/ FLATC_EXE ${flatc_location} @@ -154,10 +154,16 @@ set(binary_parser_sources ) add_library(rl_binary_parser STATIC ${binary_parser_headers} ${binary_parser_sources}) +set_target_properties(rl_binary_parser PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(WIN32) + set_target_properties(rl_binary_parser PROPERTIES DEBUG_POSTFIX d) +endif() + target_link_libraries(rl_binary_parser PUBLIC vw_core RapidJSON PRIVATE libzstd_static) target_include_directories(rl_binary_parser PUBLIC ${CMAKE_CURRENT_LIST_DIR}/ + ${CMAKE_CURRENT_LIST_DIR}/generated/v2/ ${CMAKE_CURRENT_LIST_DIR}/../ext_libs/zstd/lib/ ${CMAKE_CURRENT_LIST_DIR}/../ext_libs/date/ ) @@ -169,11 +175,15 @@ if(TARGET flatbuffers::flatbuffers) else() target_include_directories(rl_binary_parser PRIVATE ${FLATBUFFERS_INCLUDE_DIR}) endif() -add_dependencies(rl_binary_parser fbgen) +add_dependencies(rl_binary_parser fbgen_external_parser) add_executable(rl_binary_parser_bin main.cc) target_link_libraries(rl_binary_parser_bin PUBLIC rl_binary_parser) -set_target_properties(rl_binary_parser_bin PROPERTIES OUTPUT_NAME "vw") +if (NOT rlclientlib_BUILD_DOTNET) + # The build for .NET bindings configures all binary output to a single directory + # In this case we can't name the binary parser "vw", since this will overwrite the real vw executable + set_target_properties(rl_binary_parser_bin PROPERTIES OUTPUT_NAME "vw") +endif() if(STATIC_LINK_BINARY_PARSER AND NOT APPLE) target_link_libraries(rl_binary_parser_bin PRIVATE -static) From bc45f322027a7ac72224a444b52eed8c2c21faa6 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 14:20:36 -0400 Subject: [PATCH 02/15] constants --- include/constants.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/include/constants.h b/include/constants.h index 29c770e8b..9e2f41a58 100644 --- a/include/constants.h +++ b/include/constants.h @@ -107,6 +107,15 @@ const char* const MODEL_FILE_NAME = "model_file_loader.file_name"; const char* const MODEL_FILE_MUST_EXIST = "model_file_loader.file_must_exist"; const char* const ZSTD_COMPRESSION_LEVEL = "zstd.compression_level"; + +#ifdef RL_BUILD_FEDERATION +// local joiner for federated learning +const char* const JOINER_EUD_DURATION = "eud.duration"; +const char* const JOINER_PROBLEM_TYPE = "joiner.problem.type"; +const char* const JOINER_REWARD_FUNCTION = "joiner.reward.function"; +const char* const JOINER_LEARNING_MODE = "joiner.learning.mode"; +#endif + } // namespace name } // namespace reinforcement_learning @@ -149,6 +158,30 @@ const int DEFAULT_VW_POOL_INIT_SIZE = 4; const int DEFAULT_PROTOCOL_VERSION = 1; const char* const DEFAULT_AUDIT_OUTPUT_PATH = "audit"; +#ifdef RL_BUILD_FEDERATION +// Configuration values for local joiner +const char* const PROBLEM_TYPE_UNKNOWN = "PROBLEM_TYPE_UNKNOWN"; +const char* const PROBLEM_TYPE_CB = "PROBLEM_TYPE_CB"; +const char* const PROBLEM_TYPE_CCB = "PROBLEM_TYPE_CCB"; +const char* const PROBLEM_TYPE_SLATES = "PROBLEM_TYPE_SLATES"; +const char* const PROBLEM_TYPE_CA = "PROBLEM_TYPE_CA"; +const char* const PROBLEM_TYPE_MULTISTEP = "PROBLEM_TYPE_MULTISTEP"; +const char* const REWARD_FUNCTION_EARLIEST = "REWARD_FUNCTION_EARLIEST"; +const char* const REWARD_FUNCTION_AVERAGE = "REWARD_FUNCTION_AVERAGE"; +const char* const REWARD_FUNCTION_MEDIAN = "REWARD_FUNCTION_MEDIAN"; +const char* const REWARD_FUNCTION_SUM = "REWARD_FUNCTION_SUM"; +const char* const REWARD_FUNCTION_MIN = "REWARD_FUNCTION_MEAN"; +const char* const REWARD_FUNCTION_MAX = "REWARD_FUNCTION_MAX"; +#endif + +// These are outside of #ifdef section so that we can recognize them as invalid +// configuration options when rlclientlib is compiled without RL_BUILD_FEDERATION +// +// Use local_loop_controller for model data +const char* const LOCAL_LOOP_MODEL_DATA = "LOCAL_LOOP_MODEL_DATA"; +// Send events to local_loop_controller +const char* const LOCAL_LOOP_SENDER = "LOCAL_LOOP_SENDER"; + const char* get_default_episode_sender(); const char* get_default_observation_sender(); const char* get_default_interaction_sender(); From a0458d20c6edfc57463290c3badba6f0fbcc419e Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 15:14:47 -0400 Subject: [PATCH 03/15] no v2 namespace synonym exposure in headers --- .../event_processors/joined_event.h | 25 +++++--- external_parser/event_processors/loop.h | 9 ++- external_parser/event_processors/metadata.h | 8 +-- external_parser/event_processors/reward.h | 2 +- .../event_processors/timestamp_helper.cc | 2 + .../event_processors/timestamp_helper.h | 10 +-- .../event_processors/typed_events.h | 63 ++++++++++++------- external_parser/joiners/example_joiner.cc | 2 + external_parser/joiners/example_joiner.h | 36 +++++++---- external_parser/joiners/i_joiner.h | 13 ++-- .../joiners/multistep_example_joiner.cc | 2 + .../joiners/multistep_example_joiner.h | 25 ++++---- external_parser/log_converter.cc | 1 + external_parser/log_converter.h | 2 - external_parser/parse_example_binary.cc | 2 + external_parser/parse_example_converter.cc | 2 +- external_parser/parse_example_converter.h | 3 +- external_parser/parse_example_external.cc | 2 + external_parser/unit_tests/test_common.cc | 2 - .../unit_tests/test_timestamp_helper.cc | 9 +-- 20 files changed, 134 insertions(+), 86 deletions(-) diff --git a/external_parser/event_processors/joined_event.h b/external_parser/event_processors/joined_event.h index 61d0db795..8439d5403 100644 --- a/external_parser/event_processors/joined_event.h +++ b/external_parser/event_processors/joined_event.h @@ -10,8 +10,6 @@ // clang-format on #include "vw/json_parser/parse_example_json.h" -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - namespace joined_event { struct MultiSlotInteraction @@ -132,8 +130,11 @@ struct cb_joined_event : public typed_joined_event // original reward is used to record the observed reward of apprentice mode original_reward = reward_function(outcome_events, default_reward); - if (interaction_metadata.learning_mode == v2::LearningModeType_Apprentice) { set_apprentice_reward(); } - else { reward = original_reward; } + if (interaction_metadata.learning_mode == reinforcement_learning::messages::flatbuff::v2::LearningModeType_Apprentice) { + set_apprentice_reward(); + } else { + reward = original_reward; + } } void calculate_metrics(VW::details::dsjson_metrics* metrics) override @@ -228,7 +229,7 @@ struct ccb_joined_event : public typed_joined_event { size_t num_of_slots = multi_slot_interaction.interaction_data.size(); - if (metadata_info.learning_mode == v2::LearningModeType_Apprentice && + if (metadata_info.learning_mode == reinforcement_learning::messages::flatbuff::v2::LearningModeType_Apprentice && num_of_slots != multi_slot_interaction.baseline_actions.size()) { logger.out_error( @@ -242,7 +243,7 @@ struct ccb_joined_event : public typed_joined_event { for (auto& outcome : outcome_events) { - if (outcome.index_type == v2::IndexValue_literal && !outcome.s_index.empty()) + if (outcome.index_type == reinforcement_learning::messages::flatbuff::v2::IndexValue_literal && !outcome.s_index.empty()) { auto iterator = slot_id_to_index_map.find(outcome.s_index); if (iterator != slot_id_to_index_map.end()) { outcome.index = iterator->second; } @@ -278,8 +279,12 @@ struct ccb_joined_event : public typed_joined_event } } - if (metadata_info.learning_mode == v2::LearningModeType_Apprentice) { set_apprentice_reward(); } - else { rewards.assign(original_rewards.begin(), original_rewards.end()); } + if (metadata_info.learning_mode == reinforcement_learning::messages::flatbuff::v2::LearningModeType_Apprentice) { + set_apprentice_reward(); + } + else { + rewards.assign(original_rewards.begin(), original_rewards.end()); + } } void calculate_metrics(VW::details::dsjson_metrics* metrics) override @@ -362,7 +367,7 @@ struct slates_joined_event : public typed_joined_event reward = default_reward; original_reward = reward_function(outcome_events, default_reward); - if (metadata_info.learning_mode == v2::LearningModeType_Apprentice) + if (metadata_info.learning_mode == reinforcement_learning::messages::flatbuff::v2::LearningModeType_Apprentice) { logger.out_warn("Apprentice mode is not implmeneted for slates."); } @@ -447,7 +452,7 @@ struct ca_joined_event : public typed_joined_event // original reward is used to record the observed reward of apprentice mode original_reward = reward_function(outcome_events, default_reward); - if (interaction_metadata.learning_mode == v2::LearningModeType_Apprentice) + if (interaction_metadata.learning_mode == reinforcement_learning::messages::flatbuff::v2::LearningModeType_Apprentice) { logger.out_warn("Apprentice mode is not implmeneted for cats."); } diff --git a/external_parser/event_processors/loop.h b/external_parser/event_processors/loop.h index 355174c06..b853caf9d 100644 --- a/external_parser/event_processors/loop.h +++ b/external_parser/event_processors/loop.h @@ -3,8 +3,6 @@ // FileFormat_generated.h used for the payload type and encoding enum's #include "generated/v2/FileFormat_generated.h" -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - namespace loop { template @@ -39,9 +37,10 @@ class sticky_value struct loop_info { sticky_value default_reward = sticky_value(0.f); - sticky_value learning_mode_config = - sticky_value(v2::LearningModeType_Online); - sticky_value problem_type_config; + sticky_value learning_mode_config = + sticky_value( + reinforcement_learning::messages::flatbuff::v2::LearningModeType_Online); + sticky_value problem_type_config; sticky_value use_client_time = sticky_value(false); bool is_configured() const diff --git a/external_parser/event_processors/metadata.h b/external_parser/event_processors/metadata.h index 28f8f9f51..e120f750d 100644 --- a/external_parser/event_processors/metadata.h +++ b/external_parser/event_processors/metadata.h @@ -4,18 +4,16 @@ #include "generated/v2/FileFormat_generated.h" #include "timestamp_helper.h" -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - namespace metadata { // used both for interactions and observations struct event_metadata_info { std::string app_id; - v2::PayloadType payload_type; + reinforcement_learning::messages::flatbuff::v2::PayloadType payload_type; float pass_probability; - v2::EventEncoding event_encoding; + reinforcement_learning::messages::flatbuff::v2::EventEncoding event_encoding; std::string event_id; - v2::LearningModeType learning_mode; + reinforcement_learning::messages::flatbuff::v2::LearningModeType learning_mode; }; } // namespace metadata diff --git a/external_parser/event_processors/reward.h b/external_parser/event_processors/reward.h index a85e852d5..c71a8e6b8 100644 --- a/external_parser/event_processors/reward.h +++ b/external_parser/event_processors/reward.h @@ -12,7 +12,7 @@ struct outcome_event { } metadata::event_metadata_info metadata; - v2::IndexValue index_type; + reinforcement_learning::messages::flatbuff::v2::IndexValue index_type; std::string s_index; int index; std::string s_value; diff --git a/external_parser/event_processors/timestamp_helper.cc b/external_parser/event_processors/timestamp_helper.cc index 1ea24e45e..26c969145 100644 --- a/external_parser/event_processors/timestamp_helper.cc +++ b/external_parser/event_processors/timestamp_helper.cc @@ -2,6 +2,8 @@ #include "vw/io/logger.h" +namespace v2 = reinforcement_learning::messages::flatbuff::v2; + TimePoint timestamp_to_chrono(const v2::TimeStamp& ts) { // --- date transformation --- diff --git a/external_parser/event_processors/timestamp_helper.h b/external_parser/event_processors/timestamp_helper.h index b6b2e31eb..ad28a38e6 100644 --- a/external_parser/event_processors/timestamp_helper.h +++ b/external_parser/event_processors/timestamp_helper.h @@ -6,9 +6,9 @@ #include -namespace v2 = reinforcement_learning::messages::flatbuff::v2; using TimePoint = std::chrono::time_point; -TimePoint timestamp_to_chrono(const v2::TimeStamp& ts); -bool is_empty_timestamp(const v2::TimeStamp& ts); -TimePoint get_enqueued_time(const v2::TimeStamp* enqueued_time_utc, const v2::TimeStamp* client_time_utc, - bool use_client_time, VW::io::logger& logger); \ No newline at end of file +TimePoint timestamp_to_chrono(const reinforcement_learning::messages::flatbuff::v2::TimeStamp& ts); +bool is_empty_timestamp(const reinforcement_learning::messages::flatbuff::v2::TimeStamp& ts); +TimePoint get_enqueued_time(const reinforcement_learning::messages::flatbuff::v2::TimeStamp* enqueued_time_utc, + const reinforcement_learning::messages::flatbuff::v2::TimeStamp* client_time_utc, bool use_client_time, + VW::io::logger& logger); \ No newline at end of file diff --git a/external_parser/event_processors/typed_events.h b/external_parser/event_processors/typed_events.h index 4737852b8..0d71cb352 100644 --- a/external_parser/event_processors/typed_events.h +++ b/external_parser/event_processors/typed_events.h @@ -8,16 +8,14 @@ #include "loop.h" #include "zstd.h" -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - namespace typed_event { template struct event_processor; template <> -struct event_processor +struct event_processor { - static bool is_valid(const v2::MultiSlotEvent& evt, const loop::loop_info& loop_info, VW::io::logger& logger) + static bool is_valid(const reinforcement_learning::messages::flatbuff::v2::MultiSlotEvent& evt, const loop::loop_info& loop_info, VW::io::logger& logger) { if (evt.context() == nullptr || evt.slots() == nullptr) { return false; } @@ -33,18 +31,24 @@ struct event_processor return true; } - static v2::LearningModeType get_learning_mode(const v2::MultiSlotEvent& evt) { return evt.learning_mode(); } + static reinforcement_learning::messages::flatbuff::v2::LearningModeType get_learning_mode( + const reinforcement_learning::messages::flatbuff::v2::MultiSlotEvent& evt) + { + return evt.learning_mode(); + } - static std::string get_context(const v2::MultiSlotEvent& evt) + static std::string get_context(const reinforcement_learning::messages::flatbuff::v2::MultiSlotEvent& evt) { return {reinterpret_cast(evt.context()->data()), evt.context()->size()}; } - static joined_event::joined_event fill_in_joined_event(const v2::MultiSlotEvent& evt, const v2::Metadata& metadata, - const TimePoint& enqueued_time_utc, std::string&& line_vec) + static joined_event::joined_event fill_in_joined_event( + const reinforcement_learning::messages::flatbuff::v2::MultiSlotEvent& evt, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata, + const TimePoint& enqueued_time_utc, std::string&& line_vec) { joined_event::MultiSlotInteraction multislot_data; - bool is_ccb = metadata.payload_type() == v2::PayloadType_CCB; + bool is_ccb = metadata.payload_type() == reinforcement_learning::messages::flatbuff::v2::PayloadType_CCB; auto ccb_data = VW::make_unique(); auto slates_data = VW::make_unique(); @@ -95,9 +99,10 @@ struct event_processor }; template <> -struct event_processor +struct event_processor { - static bool is_valid(const v2::CbEvent& evt, const loop::loop_info& loop_info, VW::io::logger& logger) + static bool is_valid(const reinforcement_learning::messages::flatbuff::v2::CbEvent& evt, + const loop::loop_info& loop_info, VW::io::logger& logger) { if (evt.context() == nullptr || evt.action_ids() == nullptr || evt.probabilities() == nullptr) { return false; } @@ -113,15 +118,21 @@ struct event_processor return true; } - static v2::LearningModeType get_learning_mode(const v2::CbEvent& evt) { return evt.learning_mode(); } + static reinforcement_learning::messages::flatbuff::v2::LearningModeType get_learning_mode( + const reinforcement_learning::messages::flatbuff::v2::CbEvent& evt) + { + return evt.learning_mode(); + } - static std::string get_context(const v2::CbEvent& evt) + static std::string get_context(const reinforcement_learning::messages::flatbuff::v2::CbEvent& evt) { return {reinterpret_cast(evt.context()->data()), evt.context()->size()}; } static joined_event::joined_event fill_in_joined_event( - const v2::CbEvent& evt, const v2::Metadata& metadata, const TimePoint& enqueued_time_utc, std::string&& line_vec) + const reinforcement_learning::messages::flatbuff::v2::CbEvent& evt, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata, + const TimePoint& enqueued_time_utc, std::string&& line_vec) { auto cb_data = VW::make_unique(); @@ -143,9 +154,10 @@ struct event_processor }; template <> -struct event_processor +struct event_processor { - static bool is_valid(const v2::CaEvent& evt, const loop::loop_info& loop_info, VW::io::logger& logger) + static bool is_valid(const reinforcement_learning::messages::flatbuff::v2::CaEvent& evt, + const loop::loop_info& loop_info, VW::io::logger& logger) { if (evt.context() == nullptr) { return false; } @@ -161,15 +173,21 @@ struct event_processor return true; } - static v2::LearningModeType get_learning_mode(const v2::CaEvent& evt) { return evt.learning_mode(); } + static reinforcement_learning::messages::flatbuff::v2::LearningModeType get_learning_mode( + const reinforcement_learning::messages::flatbuff::v2::CaEvent& evt) + { + return evt.learning_mode(); + } - static std::string get_context(const v2::CaEvent& evt) + static std::string get_context(const reinforcement_learning::messages::flatbuff::v2::CaEvent& evt) { return {reinterpret_cast(evt.context()->data()), evt.context()->size()}; } static joined_event::joined_event fill_in_joined_event( - const v2::CaEvent& evt, const v2::Metadata& metadata, const TimePoint& enqueued_time_utc, std::string&& line_vec) + const reinforcement_learning::messages::flatbuff::v2::CaEvent& evt, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata, + const TimePoint& enqueued_time_utc, std::string&& line_vec) { auto ca_data = VW::make_unique(); ca_data->interaction_data.event_id = metadata.id()->str(); @@ -186,10 +204,11 @@ struct event_processor }; template -bool process_compression(const uint8_t* data, size_t size, const v2::Metadata& metadata, const T*& payload, - flatbuffers::DetachedBuffer& detached_buffer, VW::io::logger& logger) +bool process_compression(const uint8_t* data, size_t size, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata, const T*& payload, + flatbuffers::DetachedBuffer& detached_buffer, VW::io::logger& logger) { - if (metadata.encoding() == v2::EventEncoding_Zstd) + if (metadata.encoding() == reinforcement_learning::messages::flatbuff::v2::EventEncoding_Zstd) { size_t buff_size = ZSTD_getFrameContentSize(data, size); if (buff_size == ZSTD_CONTENTSIZE_ERROR) diff --git a/external_parser/joiners/example_joiner.cc b/external_parser/joiners/example_joiner.cc index 4ff046a24..538b197c5 100644 --- a/external_parser/joiners/example_joiner.cc +++ b/external_parser/joiners/example_joiner.cc @@ -30,6 +30,8 @@ #include "vw/core/parser.h" #include "vw/core/scope_exit.h" +namespace v2 = reinforcement_learning::messages::flatbuff::v2; + example_joiner::example_joiner(VW::workspace* vw) : i_joiner(vw->logger), _vw(vw), _reward_calculation(&reward::earliest), _binary_to_json(false) { diff --git a/external_parser/joiners/example_joiner.h b/external_parser/joiners/example_joiner.h index 89073f2d3..a317d0357 100644 --- a/external_parser/joiners/example_joiner.h +++ b/external_parser/joiners/example_joiner.h @@ -23,23 +23,32 @@ class example_joiner : public i_joiner ~example_joiner() override; - void set_reward_function(v2::RewardFunctionType type, bool sticky = false) override; + void set_reward_function( + reinforcement_learning::messages::flatbuff::v2::RewardFunctionType type, bool sticky = false) override; void set_default_reward(float default_reward, bool sticky = false) override; - void set_learning_mode_config(v2::LearningModeType learning_mode, bool sticky = false) override; - void set_problem_type_config(v2::ProblemType problem_type, bool sticky = false) override; + void set_learning_mode_config( + reinforcement_learning::messages::flatbuff::v2::LearningModeType learning_mode, bool sticky = false) override; + void set_problem_type_config( + reinforcement_learning::messages::flatbuff::v2::ProblemType problem_type, bool sticky = false) override; void set_use_client_time(bool use_client_time, bool sticky = false) override; void apply_cli_overrides(VW::workspace* all, const VW::external::parser_options& parsed_options) override; bool joiner_ready() override; float default_reward() const { return _loop_info.default_reward; } - v2::LearningModeType learning_mode_config() const { return _loop_info.learning_mode_config; } - v2::ProblemType problem_type_config() const { return _loop_info.problem_type_config; } + reinforcement_learning::messages::flatbuff::v2::LearningModeType learning_mode_config() const + { + return _loop_info.learning_mode_config; + } + reinforcement_learning::messages::flatbuff::v2::ProblemType problem_type_config() const + { + return _loop_info.problem_type_config; + } bool use_client_time() const { return _loop_info.use_client_time; } // Takes an event which will have a timestamp and event payload // groups all events interactions with their event observations based on their // id. The grouped events can be processed when process_joined() is called - bool process_event(const v2::JoinedEvent& joined_event) override; + bool process_event(const reinforcement_learning::messages::flatbuff::v2::JoinedEvent& joined_event) override; /** * Takes all grouped events, processes them (e.g. decompression) and populates @@ -90,12 +99,16 @@ class example_joiner : public i_joiner void persist_metrics(VW::metric_sink& sink) override; private: - bool process_dedup(const v2::Event& event, const v2::Metadata& metadata); + bool process_dedup(const reinforcement_learning::messages::flatbuff::v2::Event& event, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata); - bool process_interaction( - const v2::Event& event, const v2::Metadata& metadata, const TimePoint& enqueued_time_utc, VW::multi_ex& examples); + bool process_interaction(const reinforcement_learning::messages::flatbuff::v2::Event& event, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata, + const TimePoint& enqueued_time_utc, VW::multi_ex& examples); - bool process_outcome(const v2::Event& event, const v2::Metadata& metadata, const TimePoint& enqueued_time_utc); + bool process_outcome(const reinforcement_learning::messages::flatbuff::v2::Event& event, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata, + const TimePoint& enqueued_time_utc); void clear_batch_info(); void clear_event_id_batch_info(const std::string& id); @@ -115,7 +128,8 @@ class example_joiner : public i_joiner // (multi)example std::unordered_map _batch_grouped_examples; // from event id to all the events that have that event id - std::unordered_map> _batch_grouped_events; + std::unordered_map> + _batch_grouped_events; std::queue _batch_event_order; std::vector _example_pool; diff --git a/external_parser/joiners/i_joiner.h b/external_parser/joiners/i_joiner.h index 5561b1cc8..f6f786cf2 100644 --- a/external_parser/joiners/i_joiner.h +++ b/external_parser/joiners/i_joiner.h @@ -21,18 +21,19 @@ #include "vw/core/vw.h" // clang-format on -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - class i_joiner { public: explicit i_joiner(VW::io::logger logger_) : logger(std::move(logger_)) {} virtual ~i_joiner() = default; - virtual void set_reward_function(const v2::RewardFunctionType type, bool sticky = false) = 0; + virtual void set_reward_function( + const reinforcement_learning::messages::flatbuff::v2::RewardFunctionType type, bool sticky = false) = 0; virtual void set_default_reward(float default_reward, bool sticky = false) = 0; - virtual void set_learning_mode_config(v2::LearningModeType learning_mode, bool sticky = false) = 0; - virtual void set_problem_type_config(v2::ProblemType problem_type, bool sticky = false) = 0; + virtual void set_learning_mode_config( + reinforcement_learning::messages::flatbuff::v2::LearningModeType learning_mode, bool sticky = false) = 0; + virtual void set_problem_type_config( + reinforcement_learning::messages::flatbuff::v2::ProblemType problem_type, bool sticky = false) = 0; virtual void set_use_client_time(bool use_client_time, bool sticky = false) = 0; virtual void apply_cli_overrides(VW::workspace* all, const VW::external::parser_options& parsed_options) = 0; @@ -47,7 +48,7 @@ class i_joiner // Takes an event which will have a timestamp and event payload // groups all events interactions with their event observations based on their // id. The grouped events can be processed when process_joined() is called - virtual bool process_event(const v2::JoinedEvent& joined_event) = 0; + virtual bool process_event(const reinforcement_learning::messages::flatbuff::v2::JoinedEvent& joined_event) = 0; // Takes all grouped events, processes them (e.g. decompression) and populates // the examples array with complete example(s) ready to be used by vw for // training diff --git a/external_parser/joiners/multistep_example_joiner.cc b/external_parser/joiners/multistep_example_joiner.cc index 8450ae0dd..c06be435c 100644 --- a/external_parser/joiners/multistep_example_joiner.cc +++ b/external_parser/joiners/multistep_example_joiner.cc @@ -24,6 +24,8 @@ #include "vw/core/v_array.h" #include "vw/io/logger.h" +namespace v2 = reinforcement_learning::messages::flatbuff::v2; + multistep_example_joiner::multistep_example_joiner(VW::workspace* vw) : i_joiner(vw->logger) , _vw(vw) diff --git a/external_parser/joiners/multistep_example_joiner.h b/external_parser/joiners/multistep_example_joiner.h index 708ee7b0d..b745a4d0c 100644 --- a/external_parser/joiners/multistep_example_joiner.h +++ b/external_parser/joiners/multistep_example_joiner.h @@ -23,8 +23,6 @@ #include "vw/core/vw.h" // clang-format on -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - enum multistep_reward_funtion_type { Identity = 0, @@ -61,17 +59,20 @@ class multistep_example_joiner : public i_joiner ~multistep_example_joiner() override; - void set_reward_function(const v2::RewardFunctionType type, bool sticky) override; + void set_reward_function( + const reinforcement_learning::messages::flatbuff::v2::RewardFunctionType type, bool sticky) override; void set_default_reward(float default_reward, bool sticky) override; - void set_learning_mode_config(v2::LearningModeType learning_mode, bool sticky) override; - void set_problem_type_config(v2::ProblemType problem_type, bool sticky) override; + void set_learning_mode_config( + reinforcement_learning::messages::flatbuff::v2::LearningModeType learning_mode, bool sticky) override; + void set_problem_type_config( + reinforcement_learning::messages::flatbuff::v2::ProblemType problem_type, bool sticky) override; void set_use_client_time(bool use_client_time, bool sticky = false) override; void apply_cli_overrides(VW::workspace* all, const VW::external::parser_options& parsed_options) override; bool joiner_ready() override; bool current_event_is_skip_learn() override; - bool process_event(const v2::JoinedEvent& joined_event) override; + bool process_event(const reinforcement_learning::messages::flatbuff::v2::JoinedEvent& joined_event) override; bool process_joined(VW::multi_ex& examples) override; bool processing_batch() override; @@ -84,17 +85,18 @@ class multistep_example_joiner : public i_joiner struct Parsed { const TimePoint timestamp; - const v2::Metadata& meta; + const reinforcement_learning::messages::flatbuff::v2::Metadata& meta; const event_t& event; }; void set_multistep_reward_function(const multistep_reward_funtion_type type, bool sticky); private: bool populate_order(); - reward::outcome_event process_outcome( - const TimePoint& timestamp, const v2::Metadata& metadata, const v2::OutcomeEvent& event); + reward::outcome_event process_outcome(const TimePoint& timestamp, + const reinforcement_learning::messages::flatbuff::v2::Metadata& metadata, + const reinforcement_learning::messages::flatbuff::v2::OutcomeEvent& event); joined_event::joined_event process_interaction( - const Parsed& event_meta, VW::multi_ex& examples, float reward); + const Parsed& event_meta, VW::multi_ex& examples, float reward); void populate_episodic_rewards(); private: @@ -107,7 +109,8 @@ class multistep_example_joiner : public i_joiner loop::sticky_value _multistep_reward_calculation; loop::loop_info _loop_info; - std::unordered_map>> _interactions; + std::unordered_map>> + _interactions; std::unordered_map> _outcomes; std::vector _episodic_outcomes; diff --git a/external_parser/log_converter.cc b/external_parser/log_converter.cc index e1227996d..21e88d2be 100644 --- a/external_parser/log_converter.cc +++ b/external_parser/log_converter.cc @@ -9,6 +9,7 @@ namespace log_converter { namespace rj = rapidjson; +namespace v2 = reinforcement_learning::messages::flatbuff::v2; void build_json(std::ofstream& outfile, joined_event::joined_event& je, VW::io::logger& logger) { diff --git a/external_parser/log_converter.h b/external_parser/log_converter.h index 14eb49fc1..444ef1597 100644 --- a/external_parser/log_converter.h +++ b/external_parser/log_converter.h @@ -10,8 +10,6 @@ #include #include -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - namespace log_converter { void build_json(std::ofstream& outfile, joined_event::joined_event& je, VW::io::logger& logger); diff --git a/external_parser/parse_example_binary.cc b/external_parser/parse_example_binary.cc index d6fbc4ea9..a5eb81b11 100644 --- a/external_parser/parse_example_binary.cc +++ b/external_parser/parse_example_binary.cc @@ -19,6 +19,8 @@ #include #include +namespace v2 = reinforcement_learning::messages::flatbuff::v2; + // TODO need to check if errors will be detected from stderr/stdout/other and // use appropriate logger diff --git a/external_parser/parse_example_converter.cc b/external_parser/parse_example_converter.cc index bcc38f346..17eafaa08 100644 --- a/external_parser/parse_example_converter.cc +++ b/external_parser/parse_example_converter.cc @@ -11,7 +11,7 @@ namespace VW { namespace external { -binary_json_converter::binary_json_converter(std::unique_ptr&& joiner, VW::io::logger logger) +binary_json_converter::binary_json_converter(std::unique_ptr&& joiner, const VW::io::logger& logger) : parser(logger), _parser(std::move(joiner), logger) { } diff --git a/external_parser/parse_example_converter.h b/external_parser/parse_example_converter.h index eab0cbe75..fa48e8c4b 100644 --- a/external_parser/parse_example_converter.h +++ b/external_parser/parse_example_converter.h @@ -15,7 +15,8 @@ namespace external class binary_json_converter : public parser { public: - binary_json_converter(std::unique_ptr&& joiner, VW::io::logger logger); // taking ownership of joiner + binary_json_converter( + std::unique_ptr&& joiner, const VW::io::logger& logger); // taking ownership of joiner ~binary_json_converter(); bool parse_examples(VW::workspace* all, io_buf& io_buf, VW::multi_ex& examples) override; void persist_metrics(metric_sink& metrics_sink) override; diff --git a/external_parser/parse_example_external.cc b/external_parser/parse_example_external.cc index 9e94abe34..8f929545d 100644 --- a/external_parser/parse_example_external.cc +++ b/external_parser/parse_example_external.cc @@ -14,6 +14,8 @@ #include #include +namespace v2 = reinforcement_learning::messages::flatbuff::v2; + namespace VW { namespace external diff --git a/external_parser/unit_tests/test_common.cc b/external_parser/unit_tests/test_common.cc index 8481fbaa3..baabc5e11 100644 --- a/external_parser/unit_tests/test_common.cc +++ b/external_parser/unit_tests/test_common.cc @@ -4,8 +4,6 @@ #include "vw/core/parser.h" #include "vw/io/io_adapter.h" -namespace v2 = reinforcement_learning::messages::flatbuff::v2; - namespace endian { bool is_big_endian(void) diff --git a/external_parser/unit_tests/test_timestamp_helper.cc b/external_parser/unit_tests/test_timestamp_helper.cc index 22cde6122..f892f6f51 100644 --- a/external_parser/unit_tests/test_timestamp_helper.cc +++ b/external_parser/unit_tests/test_timestamp_helper.cc @@ -8,16 +8,17 @@ // copied from there (hasn't changed since it was checked in) // since we need the internals to check that time transformations are done // correctly -std::pair> gmt_now_and_timestamp() +std::pair> + gmt_now_and_timestamp() { const auto tp = std::chrono::system_clock::now(); const auto dp = date::floor(tp); const auto ymd = date::year_month_day(dp); const auto time = date::make_time(tp - dp); - return std::make_pair(v2::TimeStamp(int(ymd.year()), unsigned(ymd.month()), unsigned(ymd.day()), time.hours().count(), - time.minutes().count(), time.seconds().count(), time.subseconds().count()), - tp); + return std::make_pair(reinforcement_learning::messages::flatbuff::v2::TimeStamp(int(ymd.year()), + unsigned(ymd.month()), unsigned(ymd.day()), time.hours().count(), + time.minutes().count(), time.seconds().count(), time.subseconds().count()), tp); } BOOST_AUTO_TEST_CASE(test_later_than_timestamp) From 33987643ceb5ad927c5bf487ef7d3a7a201dff39 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 15:17:41 -0400 Subject: [PATCH 04/15] redundant moves cleanup --- external_parser/joiners/example_joiner.cc | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/external_parser/joiners/example_joiner.cc b/external_parser/joiners/example_joiner.cc index 538b197c5..c1605e395 100644 --- a/external_parser/joiners/example_joiner.cc +++ b/external_parser/joiners/example_joiner.cc @@ -578,19 +578,15 @@ void example_joiner::persist_metrics(VW::metric_sink& metrics) if (!_joiner_metrics.first_event_id.empty()) { - metrics.set_string("first_event_id", std::move(_joiner_metrics.first_event_id), true); + metrics.set_string("first_event_id", _joiner_metrics.first_event_id, true); metrics.set_string("first_event_time", - std::move( - date::format("%FT%TZ", date::floor(_joiner_metrics.first_event_timestamp))), - true); + date::format("%FT%TZ", date::floor(_joiner_metrics.first_event_timestamp)), true); } if (!_joiner_metrics.last_event_id.empty()) { - metrics.set_string("last_event_id", std::move(_joiner_metrics.last_event_id), true); + metrics.set_string("last_event_id", _joiner_metrics.last_event_id, true); metrics.set_string("last_event_time", - std::move( - date::format("%FT%TZ", date::floor(_joiner_metrics.last_event_timestamp))), - true); + date::format("%FT%TZ", date::floor(_joiner_metrics.last_event_timestamp)), true); } } } From 100d713a598f4bed0ff4283e9e2b9b850d8062e0 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 15:19:45 -0400 Subject: [PATCH 05/15] example_joiner::events_in_queue --- external_parser/joiners/example_joiner.cc | 2 ++ external_parser/joiners/example_joiner.h | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/external_parser/joiners/example_joiner.cc b/external_parser/joiners/example_joiner.cc index c1605e395..a74d11e68 100644 --- a/external_parser/joiners/example_joiner.cc +++ b/external_parser/joiners/example_joiner.cc @@ -598,6 +598,8 @@ void example_joiner::on_batch_read() {} metrics::joiner_metrics example_joiner::get_metrics() { return _joiner_metrics; } +int example_joiner::events_in_queue() { return _batch_event_order.size(); } + void example_joiner::apply_cli_overrides(VW::workspace*, const VW::external::parser_options&) {} #ifdef RL_WINDOWS_GETOBJECT_MACRO_UNDEF diff --git a/external_parser/joiners/example_joiner.h b/external_parser/joiners/example_joiner.h index a317d0357..2e9bb7b35 100644 --- a/external_parser/joiners/example_joiner.h +++ b/external_parser/joiners/example_joiner.h @@ -84,7 +84,8 @@ class example_joiner : public i_joiner // true if there are still event-groups to be processed from a deserialized // batch bool processing_batch() override; - + int events_in_queue(); + // to be called after process_joined // returns true if the event that was just processed is a skip_learn event // otherwise returns false From 6375fe5f14044caf518a863833962b7e0dc4af3d Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 15:33:07 -0400 Subject: [PATCH 06/15] BOOST_TEST_DYN_LINK for external parser test --- external_parser/unit_tests/CMakeLists.txt | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/external_parser/unit_tests/CMakeLists.txt b/external_parser/unit_tests/CMakeLists.txt index fa8ba8661..d694b2b6f 100644 --- a/external_parser/unit_tests/CMakeLists.txt +++ b/external_parser/unit_tests/CMakeLists.txt @@ -1,4 +1,3 @@ -set(Boost_USE_STATIC_LIBS ON) find_package(Boost COMPONENTS unit_test_framework filesystem system program_options thread REQUIRED) set(TEST_SOURCES @@ -32,4 +31,24 @@ target_link_libraries(binary_parser_unit_tests Boost::filesystem ) +# Automatically set BOOST_TEST_DYN_LINK if the dependency is shared. +get_target_property(boost_test_target_type Boost::unit_test_framework TYPE) +if (boost_test_target_type STREQUAL SHARED_LIBRARY) + message(STATUS "Boost::unit_test_framework looks to be a shared library. Adding BOOST_TEST_DYN_LINK") + target_compile_definitions(binary_parser_unit_tests PRIVATE BOOST_TEST_DYN_LINK) +elseif(boost_test_target_type STREQUAL UNKNOWN_LIBRARY) + # Try inferring type if vcpkg is used + if (DEFINED VCPKG_TARGET_TRIPLET) + if (VCPKG_TARGET_TRIPLET EQUAL "x64-windows" OR VCPKG_TARGET_TRIPLET EQUAL "x86-windows" OR VCPKG_TARGET_TRIPLET EQUAL "arm64-osx-dynamic" OR VCPKG_TARGET_TRIPLET EQUAL "x64-osx-dynamic") + message(STATUS "Boost::unit_test_framework looks to be a shared library based on vcpkg triplet ${VCPKG_TARGET_TRIPLET}. Adding BOOST_TEST_DYN_LINK") + target_compile_definitions(binary_parser_unit_tests PRIVATE BOOST_TEST_DYN_LINK) + endif() + # If find_package is used then by default we're looking at a shared dependency unless Boost_USE_STATIC_LIBS was set. + elseif(NOT Boost_USE_STATIC_LIBS) + message(STATUS "Boost::unit_test_framework looks to be a shared library. Adding BOOST_TEST_DYN_LINK") + target_compile_definitions(binary_parser_unit_tests PRIVATE BOOST_TEST_DYN_LINK) + endif() +endif() + + add_test(NAME binary_parser_unit_tests COMMAND binary_parser_unit_tests -- ${CMAKE_CURRENT_LIST_DIR}/test_files/) From 5c04617857cf9dd9c5911861cb3b4cdea159862d Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 15:42:54 -0400 Subject: [PATCH 07/15] minor federated client interface cleanup --- rlclientlib/federation/federated_client.h | 3 ++- rlclientlib/federation/joined_log_provider.h | 16 ++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/rlclientlib/federation/federated_client.h b/rlclientlib/federation/federated_client.h index e80df9f02..dc72be43a 100644 --- a/rlclientlib/federation/federated_client.h +++ b/rlclientlib/federation/federated_client.h @@ -42,10 +42,11 @@ struct i_federated_client * try_get_model again until report_result has been called. * * @param payload payload represents the payload to aggregate. Must be a serialized VW model delta. + * @param size payload represents the size of the payload to aggregate. Must be a serialized VW model delta. * @param status Contains error information in the event of a failure * @returns Status code */ - RL_ATTR(nodiscard) virtual int report_result(const std::vector& payload, api_status* status = nullptr) = 0; + RL_ATTR(nodiscard) virtual int report_result(const uint8_t* payload, size_t size, api_status* status = nullptr) = 0; }; } // namespace reinforcement_learning diff --git a/rlclientlib/federation/joined_log_provider.h b/rlclientlib/federation/joined_log_provider.h index b2de11cee..261fa9d44 100644 --- a/rlclientlib/federation/joined_log_provider.h +++ b/rlclientlib/federation/joined_log_provider.h @@ -1,6 +1,7 @@ #pragma once #include "api_status.h" +#include "data_buffer.h" #include "future_compat.h" #include "vw/io/io_adapter.h" @@ -10,15 +11,6 @@ namespace reinforcement_learning { -struct i_joined_log_batch -{ - virtual ~i_joined_log_batch() = default; - - /// Returns next chunk of batch. chunk_reader will be nullptr when then batch is complete. - RL_ATTR(nodiscard) - virtual int next(std::unique_ptr& chunk_reader, api_status* status = nullptr) = 0; -}; - /** * @brief This interface allows polling access to logged event data. */ @@ -26,9 +18,9 @@ struct i_joined_log_provider { virtual ~i_joined_log_provider() = default; - /// Runs the join operation and returns the resulting batch which can be consumed. - /// The format of the data returned in the batch it implementation dependent. + // Runs the join operation and returns the resulting batch which can be consumed. + // The format of the data returned in the batch it implementation dependent. RL_ATTR(nodiscard) - virtual int invoke_join(std::unique_ptr& batch, api_status* status = nullptr) = 0; + virtual int invoke_join(std::unique_ptr& output, api_status* status = nullptr) = 0; }; } // namespace reinforcement_learning From d9038b90f82de3a62849e3afa9e3c6a9709ecbf7 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Wed, 1 Nov 2023 16:46:24 -0400 Subject: [PATCH 08/15] federation client implementation --- rlclientlib/CMakeLists.txt | 32 +- rlclientlib/federation/eud_utils.h | 67 +++ rlclientlib/federation/event_sink.h | 42 ++ rlclientlib/federation/local_client.cc | 117 +++++ rlclientlib/federation/local_client.h | 42 ++ .../federation/local_loop_controller.cc | 90 ++++ .../federation/local_loop_controller.h | 69 +++ .../federation/sender_joined_log_provider.cc | 263 +++++++++++ .../federation/sender_joined_log_provider.h | 80 ++++ rlclientlib/federation/vw_trainable_model.cc | 429 ++++++++++++++++++ rlclientlib/federation/vw_trainable_model.h | 74 +++ rlclientlib/schema/v2/Metadata.fbs | 4 +- rlclientlib/schema/v2/ModelUpdateEvent.fbs | 10 + rlclientlib/time_helper.cc | 44 +- rlclientlib/time_helper.h | 27 +- rlclientlib/utility/vw_logger_adapter.cc | 37 ++ rlclientlib/utility/vw_logger_adapter.h | 12 + 17 files changed, 1409 insertions(+), 30 deletions(-) create mode 100644 rlclientlib/federation/eud_utils.h create mode 100644 rlclientlib/federation/event_sink.h create mode 100644 rlclientlib/federation/local_client.cc create mode 100644 rlclientlib/federation/local_client.h create mode 100644 rlclientlib/federation/local_loop_controller.cc create mode 100644 rlclientlib/federation/local_loop_controller.h create mode 100644 rlclientlib/federation/sender_joined_log_provider.cc create mode 100644 rlclientlib/federation/sender_joined_log_provider.h create mode 100644 rlclientlib/federation/vw_trainable_model.cc create mode 100644 rlclientlib/federation/vw_trainable_model.h create mode 100644 rlclientlib/schema/v2/ModelUpdateEvent.fbs create mode 100644 rlclientlib/utility/vw_logger_adapter.cc create mode 100644 rlclientlib/utility/vw_logger_adapter.h diff --git a/rlclientlib/CMakeLists.txt b/rlclientlib/CMakeLists.txt index 29f405cdf..801f851cf 100644 --- a/rlclientlib/CMakeLists.txt +++ b/rlclientlib/CMakeLists.txt @@ -31,7 +31,9 @@ set(RL_FLAT_BUFFER_FILES_V2 "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/Metadata.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/MultiSlotEvent.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/MultiStepEvent.fbs" - "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/OutcomeEvent.fbs") + "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/OutcomeEvent.fbs" + "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/ProblemType.fbs" +) add_flatbuffer_schema( TARGET fbgenerator_v1 @@ -95,6 +97,7 @@ set(PROJECT_SOURCES utility/context_helper.cc utility/data_buffer.cc utility/data_buffer_streambuf.cc + utility/vw_logger_adapter.cc vw_model/pdf_model.cc vw_model/safe_vw.cc utility/stl_container_adapter.cc @@ -114,6 +117,15 @@ if(vw_USE_AZURE_FACTORIES) ) endif() +if(RL_BUILD_FEDERATION) + list(APPEND PROJECT_SOURCES + federation/local_client.cc + federation/local_loop_controller.cc + federation/sender_joined_log_provider.cc + federation/vw_trainable_model.cc + ) +endif() + set(PROJECT_PUBLIC_HEADERS ../include/action_flags.h ../include/api_status.h @@ -176,6 +188,7 @@ set(PROJECT_PRIVATE_HEADERS utility/object_pool.h utility/periodic_background_proc.h utility/watchdog.h + utility/vw_logger_adapter.h vw_model/pdf_model.h vw_model/safe_vw.h vw_model/vw_model.h @@ -193,6 +206,18 @@ if(vw_USE_AZURE_FACTORIES) ) endif() +if(RL_BUILD_FEDERATION) + list(APPEND PROJECT_PRIVATE_HEADERS + federation/event_sink.h + federation/federated_client.h + federation/joined_log_provider.h + federation/local_client.h + federation/local_loop_controller.h + federation/sender_joined_log_provider.h + federation/vw_trainable_model.h + ) +endif() + source_group("Sources" FILES ${PROJECT_SOURCES}) source_group("Public headers" FILES ${PROJECT_PUBLIC_HEADERS}) source_group("Private headers" FILES ${PROJECT_PRIVATE_HEADERS}) @@ -251,6 +276,11 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") target_link_libraries(rlclientlib PUBLIC Boost::thread) endif() +# Link to external binary parser for local joining code +if (RL_BUILD_EXTERNAL_PARSER) + target_link_libraries(rlclientlib PUBLIC rl_binary_parser) +endif() + # Set paths for installing library and header files install( TARGETS rlclientlib diff --git a/rlclientlib/federation/eud_utils.h b/rlclientlib/federation/eud_utils.h new file mode 100644 index 000000000..890c4bb8a --- /dev/null +++ b/rlclientlib/federation/eud_utils.h @@ -0,0 +1,67 @@ +#pragma once + +#include "api_status.h" +#include "constants.h" +#include "err_constants.h" +#include "future_compat.h" +#include "generated/v2/Event_generated.h" +#include "generated/v2/Metadata_generated.h" +#include "joined_log_provider.h" +#include "logger/message_type.h" +#include "logger/preamble.h" +#include "rl_string_view.h" +#include "sender.h" +#include "time_helper.h" +#include "vw/common/text_utils.h" +#include "vw/io/io_adapter.h" + +#include +#include +#include +#include +#include +#include + +namespace reinforcement_learning +{ +inline int parse_int(reinforcement_learning::string_view s, int& out, reinforcement_learning::api_status* status) +{ + // can't use stol because that throws an exception. Use strtol instead. + char* end = nullptr; + int i = strtol(s.data(), &end, 10); + if (end <= s.data() && s.size() > 0) + { + out = 0; + RETURN_ERROR_ARG(nullptr, status, invalid_argument, "invalid int"); + } + out = i; + return 0; +} + +inline int parse_eud( + reinforcement_learning::string_view eud_str, std::chrono::seconds& out, reinforcement_learning::api_status* status) +{ + std::vector components; + VW::tokenize(':', eud_str, components); + if (components.size() != 3 || + std::any_of(components.begin(), components.end(), + [](const std::string& component) + { return component.empty() || !std::all_of(component.begin(), component.end(), ::isdigit); })) + { + RETURN_ERROR_ARG(nullptr, status, invalid_argument, "invalid format of eud duration"); + } + + int hours{}; + RETURN_IF_FAIL(parse_int(components[0], hours, status)); + + int minutes{}; + RETURN_IF_FAIL(parse_int(components[1], minutes, status)); + + int seconds{}; + RETURN_IF_FAIL(parse_int(components[2], seconds, status)); + + out = std::chrono::hours(hours) + std::chrono::minutes(minutes) + std::chrono::seconds(seconds); + return 0; +} + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/event_sink.h b/rlclientlib/federation/event_sink.h new file mode 100644 index 000000000..b7c025748 --- /dev/null +++ b/rlclientlib/federation/event_sink.h @@ -0,0 +1,42 @@ +#pragma once + +#include "api_status.h" +#include "data_buffer.h" +#include "future_compat.h" +#include "sender.h" + +#include + +namespace reinforcement_learning +{ +class i_event_sink +{ + using buffer = std::shared_ptr; + +public: + // Add an event batch + // Input should consist of a preamble and an EventBatch flatbuffer + RL_ATTR(nodiscard) + virtual int receive_events(const buffer& data, api_status* status = nullptr) = 0; + + // Return an object of type i_sender that will forward data to receive_events() of this object + // Each call returns a new output, and the caller of this function takes ownership of it + std::unique_ptr get_sender_proxy() { return std::unique_ptr(new sender_proxy(this)); } + + virtual ~i_event_sink() = default; + +private: + struct sender_proxy : public i_sender + { + sender_proxy(i_event_sink* event_sink) : _event_sink(event_sink) {} + virtual int v_send(const buffer& data, api_status* status = nullptr) override + { + return _event_sink->receive_events(data, status); + } + virtual int init(const utility::configuration&, api_status*) override { return error_code::success; } + virtual ~sender_proxy() = default; + i_event_sink* _event_sink; + }; +}; + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/local_client.cc b/rlclientlib/federation/local_client.cc new file mode 100644 index 000000000..9dd26b30b --- /dev/null +++ b/rlclientlib/federation/local_client.cc @@ -0,0 +1,117 @@ +#include "federation/local_client.h" + +#include "api_status.h" +#include "constants.h" +#include "err_constants.h" +#include "trace_logger.h" +#include "utility/vw_logger_adapter.h" +#include "vw/config/options_cli.h" +#include "vw/core/global_data.h" +#include "vw/core/io_buf.h" +#include "vw/core/merge.h" +#include "vw/core/parse_primitives.h" +#include "vw/core/vw.h" +#include "vw/io/io_adapter.h" + +namespace reinforcement_learning +{ +local_client::local_client(std::unique_ptr initial_model, i_trace* trace_logger) + : _current_model(std::move(initial_model)), _state(state_t::model_available), _trace_logger(trace_logger) +{ +} + +local_client::~local_client() = default; + +int local_client::try_get_model(const std::string& app_id, + /* inout */ model_management::model_data& data, /* out */ bool& model_received, api_status* status) +{ + switch (_state) + { + case state_t::model_available: + { + std::size_t pos = _current_model->id.find('/'); + assert(app_id == _current_model->id.substr(0, _current_model->id.find('/'))); + io_buf buf; + auto backing_buffer = std::make_shared>(); + buf.add_file(VW::io::create_vector_writer(backing_buffer)); + VW::save_predictor(*_current_model, buf); + auto* dest_ptr = data.alloc(backing_buffer->size()); + std::memcpy(dest_ptr, backing_buffer->data(), backing_buffer->size()); + data.increment_refresh_count(); + model_received = true; + _state = state_t::model_retrieved; + // Return current model and switch into model retrieved. + } + break; + case state_t::model_retrieved: + { + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) + << "Cannot call try_get_model again until report_result has been called."; + } + break; + default: + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) << "Invalid state."; + } + return error_code::success; +} + +int local_client::report_result(const uint8_t* payload, size_t size, api_status* status) +{ + switch (_state) + { + case state_t::model_available: + { + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) + << "Cannot call report_result again until try_get_model has been called."; + } + break; + case state_t::model_retrieved: + { + // Payload must be a delta + // Apply delta to current model and move into model available state. + auto view = VW::io::create_buffer_view(reinterpret_cast(payload), size); + auto delta = VW::model_delta::deserialize(*view); + auto new_model = *_current_model + *delta; + + // Increment iteration id for new workspace + try + { + int iteration_id = std::stoi(_current_model->id.substr(_current_model->id.find('/') + 1, std::string::npos)); + iteration_id++; + new_model->id = _current_model->id.substr(0, _current_model->id.find('/')) + "/" + std::to_string(iteration_id); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, e.what()); + } + + // Update current model + _current_model.reset(new_model.release()); + _state = state_t::model_available; + } + break; + default: + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) << "Invalid state."; + } + return error_code::success; +} + +int local_client::create(std::unique_ptr& output, const utility::configuration& config, + i_trace* trace_logger, api_status* status) +{ + std::string cmd_line = "--cb_explore_adf --json --quiet --epsilon 0.0 --first_only --id "; + cmd_line += config.get("id", "default_id"); + // Create empty model based on ML args on first call + std::string initial_command_line(config.get(name::MODEL_VW_INITIAL_COMMAND_LINE, cmd_line.c_str())); + + // TODO try catch + auto args = VW::make_unique(VW::split_command_line(initial_command_line)); + auto logger = utility::make_vw_trace_logger(trace_logger); + auto workspace = VW::initialize_experimental(std::move(args), nullptr, nullptr, nullptr, &logger); + workspace->id += "/0"; // initialize iteration id to 0 + + output = std::unique_ptr(new local_client(std::move(workspace), trace_logger)); + return error_code::success; +} + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/local_client.h b/rlclientlib/federation/local_client.h new file mode 100644 index 000000000..629964435 --- /dev/null +++ b/rlclientlib/federation/local_client.h @@ -0,0 +1,42 @@ +#pragma once + +#include "configuration.h" +#include "federation/federated_client.h" +#include "trace_logger.h" +#include "vw/core/vw_fwd.h" + +namespace reinforcement_learning +{ +class local_client : i_federated_client +{ +public: + RL_ATTR(nodiscard) + static int create(std::unique_ptr& output, const utility::configuration& config, + i_trace* trace_logger = nullptr, api_status* status = nullptr); + + RL_ATTR(nodiscard) + int try_get_model(const std::string& app_id, + /* inout */ model_management::model_data& data, /* out */ bool& model_received, + api_status* status = nullptr) override; + + RL_ATTR(nodiscard) int report_result(const uint8_t* payload, size_t size, api_status* status = nullptr) override; + + ~local_client() override; + +private: + enum class state_t + { + model_available, + model_retrieved + }; + + local_client(std::unique_ptr initial_model, i_trace* trace_logger); + + state_t _state; + std::unique_ptr _current_model; + i_trace* _trace_logger; +}; + +// Read MODEL_VW_INITIAL_COMMAND_LINE + +} // namespace reinforcement_learning \ No newline at end of file diff --git a/rlclientlib/federation/local_loop_controller.cc b/rlclientlib/federation/local_loop_controller.cc new file mode 100644 index 000000000..bf473a831 --- /dev/null +++ b/rlclientlib/federation/local_loop_controller.cc @@ -0,0 +1,90 @@ +#include "federation/local_loop_controller.h" + +#include "constants.h" +#include "err_constants.h" +#include "federation/local_client.h" +#include "federation/sender_joined_log_provider.h" +#include "model_mgmt.h" +#include "vw/io/io_adapter.h" + +namespace reinforcement_learning +{ +int local_loop_controller::create(std::unique_ptr& output, + const reinforcement_learning::utility::configuration& config, i_trace* trace_logger, api_status* status) +{ + std::string app_id = config.get(name::APP_ID, ""); + + std::unique_ptr federated_client; + std::unique_ptr trainable_model; + std::unique_ptr sender_joiner; + RETURN_IF_FAIL(local_client::create(federated_client, config, trace_logger, status)); + RETURN_IF_FAIL(trainable_vw_model::create(trainable_model, config, trace_logger, status)); + RETURN_IF_FAIL(sender_joined_log_provider::create(sender_joiner, config, trace_logger, status)); + + // sender_joiner is both an i_joined_log_provider and an i_event_sink + // we need to convert to shared_ptr and create copies for each base type + std::shared_ptr sender_joiner_shared; + std::shared_ptr joiner; + std::shared_ptr event_sink; + sender_joiner_shared = std::move(sender_joiner); + joiner = std::static_pointer_cast(sender_joiner_shared); + event_sink = std::static_pointer_cast(sender_joiner_shared); + + output = std::unique_ptr(new local_loop_controller(std::move(app_id), + std::move(federated_client), std::move(trainable_model), std::move(joiner), std::move(event_sink))); + return error_code::success; +} + +local_loop_controller::local_loop_controller(std::string app_id, std::unique_ptr&& federated_client, + std::unique_ptr&& trainable_model, std::shared_ptr&& joiner, + std::shared_ptr&& event_sink) + : _app_id(std::move(app_id)) + , _federated_client(std::move(federated_client)) + , _trainable_model(std::move(trainable_model)) + , _joiner(std::move(joiner)) + , _event_sink(std::move(event_sink)) +{ +} + +int local_loop_controller::update_global(api_status* status) +{ + // ask for a new global model + model_management::model_data data; + bool model_received = false; + RETURN_IF_FAIL(_federated_client->try_get_model(_app_id, data, model_received, status)); + + if (model_received) + { + // load the new model and immediately train it with accumulated data + RETURN_IF_FAIL(_trainable_model->set_data(data, status)); + update_local(); + + // get and send the model delta + auto buffer = std::make_shared>(); + auto writer = VW::io::create_vector_writer(buffer); + RETURN_IF_FAIL(_trainable_model->get_model_delta(*writer, status)); + + char* data_ptr = buffer->data(); + RETURN_IF_FAIL(_federated_client->report_result(reinterpret_cast(data_ptr), buffer->size(), status)); + } + return error_code::success; +} + +int local_loop_controller::update_local(api_status* status) +{ + std::unique_ptr binary_log; + RETURN_IF_FAIL(_joiner->invoke_join(binary_log, status)); + RETURN_IF_FAIL(_trainable_model->learn(std::move(binary_log), status)); + return error_code::success; +} + +int local_loop_controller::get_data(model_management::model_data& data, api_status* status) +{ + RETURN_IF_FAIL(update_global(status)); + RETURN_IF_FAIL(_trainable_model->get_data(data, status)); + return error_code::success; +} + +std::unique_ptr local_loop_controller::get_local_sender() { return _event_sink->get_sender_proxy(); } + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/local_loop_controller.h b/rlclientlib/federation/local_loop_controller.h new file mode 100644 index 000000000..093291b1a --- /dev/null +++ b/rlclientlib/federation/local_loop_controller.h @@ -0,0 +1,69 @@ +#pragma once + +#include "api_status.h" +#include "error_callback_fn.h" +#include "factory_resolver.h" +#include "federation/event_sink.h" +#include "federation/federated_client.h" +#include "federation/joined_log_provider.h" +#include "federation/vw_trainable_model.h" +#include "model_mgmt.h" +#include "sender.h" +#include "trace_logger.h" + +#include +#include + +namespace reinforcement_learning +{ +// The local_loop_controller will "plug in" to rlclientlib as an i_data_transport object. +// It exposes a get_local_sender_factory() function that creates i_sender proxy objects. +// These proxy objects will send events to its internal event sink. +// The initialization code for live_model_impl must register this factory function correctly. +class local_loop_controller : public model_management::i_data_transport +{ +public: + RL_ATTR(nodiscard) + static int create(std::unique_ptr& output, const utility::configuration& config, + i_trace* trace_logger = nullptr, api_status* status = nullptr); + + // Get model data in binary format + // This will perform joining and training on any observed events, and then return the updated model + RL_ATTR(nodiscard) + virtual int get_data(model_management::model_data& data, api_status* status = nullptr) override; + + // Returns a i_sender proxy object to be used for sending events to the internal event sink + std::unique_ptr get_local_sender(); + + virtual ~local_loop_controller() = default; + +protected: + // Constructor is private because objects should be created using the factory function + local_loop_controller(std::string app_id, std::unique_ptr&& federated_client, + std::unique_ptr&& trainable_model, std::shared_ptr&& joiner, + std::shared_ptr&& event_sink); + + // This updates global state with the federated learning server. + // If applicable, it will first report a model delta from local training. + // Then it attempts to retreive a new global model. + RL_ATTR(nodiscard) + int update_global(api_status* status = nullptr); + + // Internal implemetation to run joining and traning to update local state + RL_ATTR(nodiscard) + int update_local(api_status* status = nullptr); + + // Internal state + std::string _app_id; + std::unique_ptr _federated_client = nullptr; + std::unique_ptr _trainable_model = nullptr; + // These need to be shared_ptr because they may hold the same object + std::shared_ptr _joiner = nullptr; + std::shared_ptr _event_sink = nullptr; + + // If the federated client has received a new global model, + // we need to train on local events and upload a model delta. + bool _need_to_send_model_delta = false; +}; + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/sender_joined_log_provider.cc b/rlclientlib/federation/sender_joined_log_provider.cc new file mode 100644 index 000000000..fb6bb03fb --- /dev/null +++ b/rlclientlib/federation/sender_joined_log_provider.cc @@ -0,0 +1,263 @@ +#include "sender_joined_log_provider.h" + +#include "api_status.h" +#include "constants.h" +#include "err_constants.h" +#include "federation/eud_utils.h" +#include "future_compat.h" +#include "generated/v2/Event_generated.h" +#include "generated/v2/FileFormat_generated.h" +#include "generated/v2/Metadata_generated.h" +#include "joined_log_provider.h" +#include "logger/message_type.h" +#include "logger/preamble.h" +#include "rl_string_view.h" +#include "sender.h" +#include "time_helper.h" +#include "vw/common/text_utils.h" +#include "vw/io/io_adapter.h" + +#include + +#include +#include +#include +#include +#include +#include + +using namespace reinforcement_learning; + +namespace +{ +class buffer_reader : public VW::io::reader +{ +public: + buffer_reader(std::vector&& buffer) + : VW::io::reader(true), _buffer(std::move(buffer)), _read_head(_buffer.data()) + { + } + ~buffer_reader() override = default; + ssize_t read(char* buffer, size_t num_bytes) override + { + num_bytes = std::min((_buffer.data() + _buffer.size()) - _read_head, static_cast(num_bytes)); + if (num_bytes == 0) { return 0; } + + std::memcpy(buffer, _read_head, num_bytes); + _read_head += num_bytes; + + return num_bytes; + } + void reset() override { _read_head = _buffer.data(); } + +private: + std::vector _buffer; + uint8_t* _read_head; +}; + +timestamp fb_to_rl_timestamp(const messages::flatbuff::v2::TimeStamp& ts) +{ + return timestamp(ts.year(), ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.subsecond()); +} + +messages::flatbuff::v2::TimeStamp rl_to_fb_timestamp(const timestamp& ts) +{ + return messages::flatbuff::v2::TimeStamp(ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second, ts.sub_second); +} + +void emit_uint32(std::vector& output, uint32_t data) +{ + // TODO consider doing this a safer way + // Check endianness requirement of format and make sure we get that right here. + output.reserve(output.size() + sizeof(uint32_t)); + output.insert( + std::end(output), reinterpret_cast(&data), reinterpret_cast(&data) + sizeof(uint32_t)); +} + +int emit_filemagic_message(std::vector& output) +{ + constexpr uint32_t filemagic = 0x42465756; + constexpr uint32_t version = 1; + emit_uint32(output, filemagic); + emit_uint32(output, version); + return 0; +} + +int emit_regular_message(std::vector& output, flatbuffers::FlatBufferBuilder& fbb, + flatbuffers::Offset joined_payload) +{ + constexpr uint32_t regular = 0xFFFFFFFF; + emit_uint32(output, regular); + + fbb.Finish(joined_payload); + auto buffer = fbb.Release(); + + uint32_t size = buffer.size(); + emit_uint32(output, size); + + output.reserve(output.size() + size); + output.insert(std::end(output), buffer.data(), buffer.data() + size); + + uint32_t padding_size = size % 8; + output.reserve(output.size() + padding_size); + output.insert(std::end(output), padding_size, 0); + + return 0; +} +} // namespace + +namespace reinforcement_learning +{ +RL_ATTR(nodiscard) +int sender_joined_log_provider::create(std::unique_ptr& output, + const utility::configuration& config, i_trace* trace_logger, api_status* status) +{ + if (config.get_int(name::PROTOCOL_VERSION, 999) != 2) + { + RETURN_ERROR_LS(trace_logger, status, invalid_argument) << " protocol version 2 required"; + } + + std::string eud_duration = config.get(name::JOINER_EUD_DURATION, "UNSET"); + if (eud_duration == "UNSET") { RETURN_ERROR_ARG(trace_logger, status, invalid_argument, "eudduration must be set"); } + + std::chrono::seconds eud_offset; + RETURN_IF_FAIL(parse_eud(eud_duration, eud_offset, status)); + + output = std::unique_ptr(new sender_joined_log_provider(eud_offset, trace_logger)); + return error_code::success; +} + +sender_joined_log_provider::sender_joined_log_provider(std::chrono::seconds eud_offset, i_trace* trace_logger) + : _eud_offset(eud_offset), _trace_logger(trace_logger) +{ +} + +RL_ATTR(nodiscard) +int sender_joined_log_provider::invoke_join(std::unique_ptr& batch, api_status* status) +{ + std::lock_guard lock(_mutex); + std::vector output; + const auto eud_cutoff = std::chrono::system_clock::now() - _eud_offset; + + // Binary log starts with a FILEMAGIC header + emit_filemagic_message(output); + + for (auto interaction_iter = _interactions.cbegin(); interaction_iter != _interactions.cend();) + { + flatbuffers::FlatBufferBuilder fbb; + std::vector> joined_events; + + const auto& interaction_data = *interaction_iter; + const auto interaction_time = interaction_data._time.to_time_point(); + const auto reward_cutoff = interaction_time + _eud_offset; + + // Only process interactions that occurred before EUD cutoff time + if (interaction_time > eud_cutoff) + { + // Interactions are in std::set sorted by timestamp, so once we see the + // first event after eud_cutoff, all later events are also after eud_cutoff + break; + } + + // Add interaction to flatbuffer builder + auto interaction_fb_vec = fbb.CreateVector(interaction_data._data_ptr, interaction_data._size); + auto interaction_fb_time = rl_to_fb_timestamp(interaction_data._time); + joined_events.push_back(messages::flatbuff::v2::CreateJoinedEvent(fbb, interaction_fb_vec, &interaction_fb_time)); + + // If there are corresponding observations with the event_id, process them + const auto& observation_iter = _observations.find(interaction_data._event_id); + bool interaction_has_observations = observation_iter != _observations.end(); + + if (interaction_has_observations) + { + for (const auto& observation_data : observation_iter->second) + { + const auto observation_time = observation_data._time.to_time_point(); + if (observation_time <= reward_cutoff) + { + // Add observation to flatbuffer + auto observation_fb_vec = fbb.CreateVector(observation_data._data_ptr, observation_data._size); + auto observation_fb_time = rl_to_fb_timestamp(observation_data._time); + joined_events.push_back( + messages::flatbuff::v2::CreateJoinedEvent(fbb, observation_fb_vec, &observation_fb_time)); + } + } + } + + // Create the final flatbuffer output + auto joined_payload = messages::flatbuff::v2::CreateJoinedPayloadDirect(fbb, &joined_events); + emit_regular_message(output, fbb, joined_payload); + + // Clear data structures + _interactions.erase(interaction_iter++); + if (interaction_has_observations) { _observations.erase(observation_iter); } + } + + batch.reset(new buffer_reader(std::move(output))); + return error_code::success; +} + +int sender_joined_log_provider::receive_events(const i_sender::buffer& data_buffer, api_status* status) +{ + logger::preamble pre; + pre.read_from_bytes(data_buffer->preamble_begin(), logger::preamble::size()); + + if (pre.msg_type != logger::message_type::fb_generic_event_collection) + { + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) + << " Message type " << pre.msg_type << " cannot be handled."; + } + + // Verify the flatbuffer + auto event_batch = messages::flatbuff::v2::GetEventBatch(data_buffer->body_begin()); + flatbuffers::Verifier verifier(data_buffer->body_begin(), data_buffer->body_filled_size()); + auto result = event_batch->Verify(verifier); + + if (!result) + { + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) << "verify failed for fb_generic_event_collection"; + } + + if (event_batch->metadata()->content_encoding()->str() != "IDENTITY") + { + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) << "Can only handle IDENTITY encoding"; + } + + // Enter mutex lock + { + std::lock_guard lock(_mutex); + for (auto serialized_event : *event_batch->events()) + { + // Get the flatbuffer inside flatbuffer + const auto* serialized_payload = serialized_event->payload(); + const auto* event = flatbuffers::GetRoot(serialized_payload->data()); + + // Read flatbuffer data and emplace into the corresponding data structure + std::string event_id = event->meta()->id()->str(); + auto event_timestamp = fb_to_rl_timestamp(*event->meta()->client_time_utc()); + switch (event->meta()->payload_type()) + { + case messages::flatbuff::v2::PayloadType_CB: + case messages::flatbuff::v2::PayloadType_CCB: + case messages::flatbuff::v2::PayloadType_Slates: + case messages::flatbuff::v2::PayloadType_CA: + case messages::flatbuff::v2::PayloadType_MultiStep: + _interactions.emplace( + event_id, serialized_payload->data(), serialized_payload->size(), event_timestamp, data_buffer); + break; + + case messages::flatbuff::v2::PayloadType_Outcome: + _observations[event_id].emplace_back( + event_id, serialized_payload->data(), serialized_payload->size(), event_timestamp, data_buffer); + break; + + default: + RETURN_ERROR_LS(_trace_logger, status, invalid_argument) + << "Could not process payload type: " << event->meta()->payload_type(); + } + } + } + return error_code::success; +} + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/sender_joined_log_provider.h b/rlclientlib/federation/sender_joined_log_provider.h new file mode 100644 index 000000000..12e58457b --- /dev/null +++ b/rlclientlib/federation/sender_joined_log_provider.h @@ -0,0 +1,80 @@ +#pragma once + +#include "api_status.h" +#include "configuration.h" +#include "federation/event_sink.h" +#include "federation/joined_log_provider.h" +#include "future_compat.h" +#include "sender.h" +#include "time_helper.h" +#include "vw/io/io_adapter.h" + +//#include +#include +#include +#include +#include + +namespace reinforcement_learning +{ +class sender_joined_log_provider : public i_joined_log_provider, public i_event_sink +{ +public: + RL_ATTR(nodiscard) + static int create(std::unique_ptr& output, const utility::configuration& config, + i_trace* trace_logger = nullptr, api_status* status = nullptr); + + // Perform EUD joining on events that have previously been added + // Output is a binary joined log that can be consumed by the binary parser + RL_ATTR(nodiscard) + virtual int invoke_join(std::unique_ptr& batch, api_status* status = nullptr) override; + + // Add an event batch to the joiner + // Input should consist of a preamble and an EventBatch flatbuffer + RL_ATTR(nodiscard) + virtual int receive_events(const i_sender::buffer& data, api_status* status = nullptr) override; + + virtual ~sender_joined_log_provider() = default; + +private: + // Internal object to store event data + struct event_data + { + const std::string _event_id; + const uint8_t* _data_ptr; + const size_t _size; + const timestamp _time; + + // We must hold a copy of i_sender::buffer so that its shared_ptr doesn't go out of scope + i_sender::buffer _data_buffer; + + event_data(std::string event_id, const uint8_t* data_ptr, size_t size, timestamp time, i_sender::buffer data_buffer) + : _event_id(std::move(event_id)) + , _data_ptr(data_ptr) + , _size(size) + , _time(time) + , _data_buffer(std::move(data_buffer)) + { + } + + // Sort events by time, then by event_id + bool operator<(const event_data& other) const + { + return std::tie(_time, _event_id) < std::tie(other._time, other._event_id); + } + }; + + sender_joined_log_provider(std::chrono::seconds eud_offset, i_trace* trace_logger); + + // Set of interaction events, sorted by time + std::set _interactions; + + // Map from event_id to vector of event_data objects + std::unordered_map> _observations; + + std::chrono::seconds _eud_offset; + reinforcement_learning::i_trace* _trace_logger = nullptr; + std::mutex _mutex; +}; + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/vw_trainable_model.cc b/rlclientlib/federation/vw_trainable_model.cc new file mode 100644 index 000000000..3b3050cd9 --- /dev/null +++ b/rlclientlib/federation/vw_trainable_model.cc @@ -0,0 +1,429 @@ +#include "federation/vw_trainable_model.h" + +#include "constants.h" +#include "err_constants.h" +#include "joiners/example_joiner.h" +#include "joiners/multistep_example_joiner.h" +#include "parse_example_binary.h" +#include "str_util.h" +#include "utility/vw_logger_adapter.h" +#include "vw/config/options_cli.h" +#include "vw/core/learner.h" +#include "vw/core/parse_primitives.h" +#include "vw/core/shared_data.h" +#include "vw/core/vw.h" +#include "vw/io/logger.h" + +namespace +{ +// Helper function to train model on VW::multi_ex +// examples is cleared at the end of this function +// returns number of examples learned +int learn_and_finish_examples(VW::workspace& vw, VW::multi_ex& examples) +{ + if (examples.empty()) { return 0; } + + VW::setup_examples(vw, examples); + + if (vw.l->is_multiline()) + { + vw.learn(examples); + vw.finish_example(examples); + examples.clear(); + return 1; + } + + // single line + for (auto example : examples) { vw.learn(*example); } + for (auto example : examples) { vw.finish_example(*example); } + int size = examples.size(); + examples.clear(); + return size; +} + +// Helper function to call finish_example on VW::multi_ex +// examples is cleared at the end of this function +void finish_examples(VW::workspace& vw, VW::multi_ex& examples) +{ + if (examples.empty()) { return; } + + if (vw.l->is_multiline()) { vw.finish_example(examples); } + else + { + for (auto example : examples) { vw.finish_example(*example); } + } + + examples.clear(); +} + +} // namespace + +namespace reinforcement_learning +{ +int trainable_vw_model::create(std::unique_ptr& output, const utility::configuration& config, + i_trace* trace_logger, api_status* status) +{ + int protocol_version = config.get_int(name::PROTOCOL_VERSION, 1); + if (protocol_version != 2) + { + RETURN_ERROR_LS(trace_logger, status, invalid_argument) << "Protocol version 2 is required"; + } + + std::string command_line = config.get(name::MODEL_VW_INITIAL_COMMAND_LINE, "--quiet --preserve_performance_counters"); + std::string problem_type = config.get(name::JOINER_PROBLEM_TYPE, value::PROBLEM_TYPE_UNKNOWN); + std::string learning_mode = config.get(name::JOINER_LEARNING_MODE, value::LEARNING_MODE_ONLINE); + std::string reward_function = config.get(name::JOINER_REWARD_FUNCTION, value::REWARD_FUNCTION_EARLIEST); + + try + { + output = std::unique_ptr( + new trainable_vw_model(command_line, problem_type, learning_mode, reward_function, trace_logger)); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(trace_logger, status, model_update_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(trace_logger, status, model_update_error, "Unknown error"); + } + return error_code::success; +} + +trainable_vw_model::trainable_vw_model(std::string command_line, std::string problem_type, std::string learning_mode, + std::string reward_function, i_trace* trace_logger) + : _command_line(std::move(command_line)) + , _problem_type(std::move(problem_type)) + , _learning_mode(std::move(learning_mode)) + , _reward_function(std::move(reward_function)) + , _trace_logger(trace_logger) +{ + auto options = VW::make_unique(VW::split_command_line(_command_line)); + auto logger = utility::make_vw_trace_logger(_trace_logger); + _model = VW::initialize_experimental(std::move(options), nullptr, nullptr, nullptr, &logger); + copy_current_model_to_starting(); +} + +int trainable_vw_model::set_model(std::unique_ptr&& model, api_status* status) +{ + try + { + { + std::lock_guard lock(_mutex); + _model = std::move(model); + } + copy_current_model_to_starting(); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, "Unknown error"); + } + return error_code::success; +} + +int trainable_vw_model::set_data(const model_management::model_data& data, api_status* status) +{ + try + { + auto opts = + std::unique_ptr(new VW::config::options_cli(VW::split_command_line(_command_line))); + { + std::lock_guard lock(_mutex); + auto logger = utility::make_vw_trace_logger(_trace_logger); + _model = VW::initialize_experimental( + std::move(opts), VW::io::create_buffer_view(data.data(), data.data_sz()), nullptr, nullptr, &logger); + } + copy_current_model_to_starting(); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, "Unknown error"); + } + return error_code::success; +} + +int trainable_vw_model::get_data(model_management::model_data& data, api_status* status) +{ + try + { + int example_count = 0; + io_buf io_buffer; + auto backing_buffer = std::make_shared>(); + io_buffer.add_file(VW::io::create_vector_writer(backing_buffer)); + + { + std::lock_guard lock(_mutex); + example_count = _model->sd->weighted_labeled_examples; + VW::save_predictor(*_model, io_buffer); + } + auto* destination_buffer = data.alloc(backing_buffer->size()); + std::memcpy(destination_buffer, backing_buffer->data(), backing_buffer->size()); + data.increment_refresh_count(); + + TRACE_INFO(_trace_logger, + utility::concat("trainable_vw_model::get_data() returning model trained on ", example_count, " examples")); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, "Unknown error"); + } + return error_code::success; +} + +int trainable_vw_model::learn(std::unique_ptr&& binary_log, api_status* status) +{ + if (binary_log.get() == nullptr) + { + // TODO handle this as error? + TRACE_WARN(_trace_logger, "Received null binary log in trainable_vw_model::learn()"); + return error_code::success; + } + + try + { + std::lock_guard lock(_mutex); + + io_buf io_reader; + io_reader.add_file(std::move(binary_log)); + + std::unique_ptr joiner; + if (_problem_type == value::PROBLEM_TYPE_MULTISTEP) + { + joiner = std::unique_ptr(new multistep_example_joiner(_model.get())); + } + else { joiner = std::unique_ptr(new example_joiner(_model.get())); } + + // Set the default joiner options if no checkpoint message is present in the binary log + configure_joiner(joiner); + + VW::external::binary_parser binary_parser(std::move(joiner), utility::make_vw_trace_logger(_trace_logger)); + + int example_count = 0; + bool example_was_parsed = false; + VW::multi_ex example_out; + do { + example_out.push_back(VW::new_unused_example(*_model)); + example_was_parsed = binary_parser.parse_examples(_model.get(), io_reader, example_out); + + if (example_was_parsed) + { + auto has_newline = example_out.back()->is_newline; + if (has_newline) + { + auto last = example_out.back(); + VW::finish_example(*_model, *last); + example_out.pop_back(); + } + + example_count += learn_and_finish_examples(*_model, example_out); + } + else + { + // cleanup the unused example that the parser was called with + assert(example_out.size() == 1); + VW::finish_example(*_model, example_out); + example_out.clear(); + } + } while (example_was_parsed); + + TRACE_INFO(_trace_logger, utility::concat("trainable_vw_model::learn() learned on ", example_count, " examples")); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_rank_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(_trace_logger, status, model_rank_error, "Unknown error"); + } + return error_code::success; +} + +int trainable_vw_model::learn(VW::workspace& example_ws, VW::multi_ex& examples, api_status* status) +{ + try + { + std::lock_guard lock(_mutex); + VW::multi_ex examples_copied; + + // examples may be from a different workspace, and must be copied to this workspace + for (auto example : examples) + { + io_buf io_writer; + VW::parsers::cache::details::cache_temp_buffer temp_buffer; + auto example_buffer = std::make_shared>(); + io_writer.add_file(VW::io::create_vector_writer(example_buffer)); + VW::parsers::cache::write_example_to_cache(io_writer, example, + example_ws.parser_runtime.example_parser->lbl_parser, example_ws.runtime_state.parse_mask, temp_buffer); + io_writer.flush(); + + io_buf io_reader; + io_reader.add_file(VW::io::create_buffer_view(example_buffer->data(), example_buffer->size())); + VW::multi_ex example_out; + example_out.push_back(VW::new_unused_example(*_model)); + VW::parsers::cache::read_example_from_cache(_model.get(), io_reader, example_out); + examples_copied.insert(examples_copied.end(), example_out.begin(), example_out.end()); + } + + int example_count = learn_and_finish_examples(*_model, examples_copied); + TRACE_INFO(_trace_logger, utility::concat("trainable_vw_model::learn() learned on ", example_count, " examples")); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_rank_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(_trace_logger, status, model_rank_error, "Unknown error"); + } + return error_code::success; +} + +int trainable_vw_model::get_model_delta(VW::model_delta& output, api_status* status) +{ + try + { + int old_example_count = 0; + int new_example_count = 0; + VW::model_delta delta(nullptr); + { + std::lock_guard lock(_mutex); + old_example_count = _starting_model->sd->weighted_labeled_examples; + new_example_count = _model->sd->weighted_labeled_examples; + delta = *_model - *_starting_model; + } + copy_current_model_to_starting(); + output = std::move(delta); + + TRACE_INFO(_trace_logger, + utility::concat("trainable_vw_model::get_model_delta() created model delta with ", + new_example_count - old_example_count, " examples (current model: ", new_example_count, + ", previous model: ", old_example_count, ")")); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, "Unknown error"); + } + return error_code::success; +} + +int trainable_vw_model::get_model_delta(VW::io::writer& output, api_status* status) +{ + try + { + VW::model_delta delta(nullptr); + RETURN_IF_FAIL(get_model_delta(delta, status)); + delta.serialize(output); + } + catch (const std::exception& e) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, e.what()); + } + catch (...) + { + RETURN_ERROR_ARG(_trace_logger, status, model_update_error, "Unknown error"); + } + return error_code::success; +} + +void trainable_vw_model::copy_current_model_to_starting() +{ + auto backing_vector = std::make_shared>(); + io_buf temp_buffer; + temp_buffer.add_file(VW::io::create_vector_writer(backing_vector)); + + { + std::lock_guard lock(_mutex); + VW::save_predictor(*_model, temp_buffer); + } + + auto args = VW::split_command_line(_command_line); + if (std::find(args.begin(), args.end(), "--preserve_performance_counters") == args.end()) + { + args.emplace_back("--preserve_performance_counters"); + } + auto options = VW::make_unique(args); + + { + std::lock_guard lock(_mutex); + auto logger = utility::make_vw_trace_logger(_trace_logger); + _starting_model = VW::initialize_experimental(std::move(options), + VW::io::create_buffer_view(backing_vector->data(), backing_vector->size()), nullptr, nullptr, &logger); + } +} + +void trainable_vw_model::configure_joiner(std::unique_ptr& joiner) const +{ + if (_problem_type == value::PROBLEM_TYPE_CB) + { + joiner->set_problem_type_config(messages::flatbuff::v2::ProblemType_CB); + } + else if (_problem_type == value::PROBLEM_TYPE_CCB) + { + joiner->set_problem_type_config(messages::flatbuff::v2::ProblemType_CCB); + } + else if (_problem_type == value::PROBLEM_TYPE_SLATES) + { + joiner->set_problem_type_config(messages::flatbuff::v2::ProblemType_SLATES); + } + else if (_problem_type == value::PROBLEM_TYPE_CA) + { + joiner->set_problem_type_config(messages::flatbuff::v2::ProblemType_CA); + } + else if (_problem_type == value::PROBLEM_TYPE_MULTISTEP) + { + joiner->set_problem_type_config(messages::flatbuff::v2::ProblemType_MULTISTEP); + } + else { joiner->set_problem_type_config(messages::flatbuff::v2::ProblemType_UNKNOWN); } + + if (_learning_mode == value::LEARNING_MODE_APPRENTICE) + { + joiner->set_learning_mode_config(messages::flatbuff::v2::LearningModeType_Apprentice); + } + else if (_learning_mode == value::LEARNING_MODE_LOGGINGONLY) + { + joiner->set_learning_mode_config(messages::flatbuff::v2::LearningModeType_LoggingOnly); + } + else { joiner->set_learning_mode_config(messages::flatbuff::v2::LearningModeType_Online); } + + if (_reward_function == value::REWARD_FUNCTION_AVERAGE) + { + joiner->set_reward_function(messages::flatbuff::v2::RewardFunctionType_Average); + } + else if (_reward_function == value::REWARD_FUNCTION_MEDIAN) + { + joiner->set_reward_function(messages::flatbuff::v2::RewardFunctionType_Median); + } + else if (_reward_function == value::REWARD_FUNCTION_SUM) + { + joiner->set_reward_function(messages::flatbuff::v2::RewardFunctionType_Sum); + } + else if (_reward_function == value::REWARD_FUNCTION_MIN) + { + joiner->set_reward_function(messages::flatbuff::v2::RewardFunctionType_Min); + } + else if (_reward_function == value::REWARD_FUNCTION_MAX) + { + joiner->set_reward_function(messages::flatbuff::v2::RewardFunctionType_Max); + } + else { joiner->set_reward_function(messages::flatbuff::v2::RewardFunctionType_Earliest); } + joiner->set_default_reward(0.f); +} + +} // namespace reinforcement_learning diff --git a/rlclientlib/federation/vw_trainable_model.h b/rlclientlib/federation/vw_trainable_model.h new file mode 100644 index 000000000..cd43706c5 --- /dev/null +++ b/rlclientlib/federation/vw_trainable_model.h @@ -0,0 +1,74 @@ +#pragma once + +#include "api_status.h" +#include "configuration.h" +#include "federation/joined_log_provider.h" +#include "joiners/example_joiner.h" +#include "model_mgmt.h" +#include "trace_logger.h" +#include "vw/core/global_data.h" +#include "vw/core/merge.h" + +#include + +namespace reinforcement_learning +{ +class trainable_vw_model +{ +public: + RL_ATTR(nodiscard) + static int create(std::unique_ptr& output, const utility::configuration& config, + i_trace* trace_logger = nullptr, api_status* status = nullptr); + + // Output current model state to buffer + RL_ATTR(nodiscard) + int get_data(model_management::model_data& data, api_status* status = nullptr); + + // Overwrite internal VW model with another model + RL_ATTR(nodiscard) + int set_model(std::unique_ptr&& model, api_status* status = nullptr); + + // Overwrite internal VW model with the given model data + RL_ATTR(nodiscard) + int set_data(const model_management::model_data& data, api_status* status = nullptr); + + // Train model on data from a joined binary log + RL_ATTR(nodiscard) + int learn(std::unique_ptr&& binary_log, api_status* status = nullptr); + + // Train model on VW::example* objects + // This does not call VW::finish_example on the examples passed into here + RL_ATTR(nodiscard) + int learn(VW::workspace& example_ws, VW::multi_ex& examples, api_status* status = nullptr); + + // Generate a model_delta from the current model state and the previous call to + // get_model_delta() or set_model() or set_data() + RL_ATTR(nodiscard) + int get_model_delta(VW::model_delta& output, api_status* status = nullptr); + + RL_ATTR(nodiscard) + int get_model_delta(VW::io::writer& output, api_status* status = nullptr); + +private: + // Private constructor because we should create objects with factory function + trainable_vw_model(std::string command_line, std::string problem_type, std::string learning_mode, + std::string reward_function, i_trace* trace_logger); + + // Need to keep both current and starting model in order to create model_delta + std::unique_ptr _model = nullptr; + std::unique_ptr _starting_model = nullptr; + + void copy_current_model_to_starting(); + + const std::string _command_line; + const std::string _problem_type; + const std::string _learning_mode; + const std::string _reward_function; + + void configure_joiner(std::unique_ptr& joiner) const; + + i_trace* _trace_logger = nullptr; + std::mutex _mutex; +}; + +} // namespace reinforcement_learning diff --git a/rlclientlib/schema/v2/Metadata.fbs b/rlclientlib/schema/v2/Metadata.fbs index 095caff49..d922f462f 100644 --- a/rlclientlib/schema/v2/Metadata.fbs +++ b/rlclientlib/schema/v2/Metadata.fbs @@ -1,6 +1,6 @@ namespace reinforcement_learning.messages.flatbuff.v2; -enum PayloadType : ubyte { CB, CCB, Slates, Outcome, CA, DedupInfo, MultiStep, Episode } +enum PayloadType : ubyte { CB, CCB, Slates, Outcome, CA, DedupInfo, MultiStep, Episode, ModelUpdate } enum EventEncoding: ubyte { Identity, Zstd } struct TimeStamp { @@ -15,7 +15,7 @@ struct TimeStamp { table Metadata { id:string; - client_time_utc:TimeStamp; + client_time_utc:TimeStamp; app_id:string; payload_type:PayloadType; pass_probability:float; // Probability of event surviving throttling operation diff --git a/rlclientlib/schema/v2/ModelUpdateEvent.fbs b/rlclientlib/schema/v2/ModelUpdateEvent.fbs new file mode 100644 index 000000000..9e0a9a586 --- /dev/null +++ b/rlclientlib/schema/v2/ModelUpdateEvent.fbs @@ -0,0 +1,10 @@ +namespace reinforcement_learning.messages.flatbuff.v2; + +table ModelUpdateEvent { + // VW ModelDelta + delta: [ubyte]; + iteration_id: string; + client_id: string; +} + +root_type ModelUpdateEvent; \ No newline at end of file diff --git a/rlclientlib/time_helper.cc b/rlclientlib/time_helper.cc index 20239e84c..adc0a7923 100644 --- a/rlclientlib/time_helper.cc +++ b/rlclientlib/time_helper.cc @@ -2,16 +2,8 @@ #include "date.h" -#include #include -#include -namespace -{ -constexpr uint64_t ONE_HUNDRED_NANO_DENOMINATOR = 10000000; -using one_hundred_nano = std::ratio<1, ONE_HUNDRED_NANO_DENOMINATOR>; -using one_hundred_nanoseconds = std::chrono::duration; -} // namespace namespace reinforcement_learning { @@ -22,30 +14,34 @@ std::ostream& operator<<(std::ostream& os, const timestamp& dt) return os; } -timestamp timestamp_from_chrono(const std::chrono::time_point& tp) +timestamp::timestamp(uint16_t yr, uint8_t mo, uint8_t dy, uint8_t h, uint8_t m, uint8_t s, uint32_t ss) + : year(yr), month(mo), day(dy), hour(h), minute(m), second(s), sub_second(ss) +{ +} + +timestamp::timestamp(const std::chrono::time_point& tp) { - timestamp ts; const auto dp = date::floor(tp); const auto ymd = date::year_month_day(dp); const auto duration_since_start_of_day = tp - dp; const auto time = date::make_time(duration_since_start_of_day); - ts.year = int(ymd.year()); - ts.month = unsigned(ymd.month()); - ts.day = unsigned(ymd.day()); - ts.hour = time.hours().count(); - ts.minute = time.minutes().count(); - ts.second = static_cast(time.seconds().count()); - std::chrono::duration usec_since_start_of_day = - std::chrono::duration_cast(duration_since_start_of_day); - ts.sub_second = static_cast(usec_since_start_of_day.count() % ONE_HUNDRED_NANO_DENOMINATOR); - return ts; + auto usec_since_start_of_day = + std::chrono::duration_cast(duration_since_start_of_day); + + year = int(ymd.year()); + month = unsigned(ymd.month()); + day = unsigned(ymd.day()); + hour = time.hours().count(); + minute = time.minutes().count(); + second = static_cast(time.seconds().count()); + sub_second = static_cast(usec_since_start_of_day.count() % timestamp::one_hundred_nano::den); } -std::chrono::time_point chrono_from_timestamp(const timestamp& ts) +std::chrono::time_point timestamp::to_time_point() const { - return date::sys_days{date::year{ts.year} / ts.month / ts.day} + std::chrono::hours{ts.hour} + - std::chrono::minutes{ts.minute} + std::chrono::seconds{ts.second} + std::chrono::nanoseconds{ts.sub_second * 100}; + return date::sys_days{date::year{year} / month / day} + std::chrono::hours{hour} + std::chrono::minutes{minute} + + std::chrono::seconds{second} + timestamp::one_hundred_nanoseconds{sub_second}; } -timestamp clock_time_provider::gmt_now() { return timestamp_from_chrono(std::chrono::system_clock::now()); } +timestamp clock_time_provider::gmt_now() { return timestamp(std::chrono::system_clock::now()); } } // namespace reinforcement_learning diff --git a/rlclientlib/time_helper.h b/rlclientlib/time_helper.h index 87a638b1c..66c6ae368 100644 --- a/rlclientlib/time_helper.h +++ b/rlclientlib/time_helper.h @@ -3,12 +3,16 @@ #include #include #include +#include #include namespace reinforcement_learning { struct timestamp { + using one_hundred_nano = std::ratio<1, 10000000>; + using one_hundred_nanoseconds = std::chrono::duration; + uint16_t year = 0; // year uint8_t month = 0; // month [1-12] uint8_t day = 0; // day [1-31] @@ -16,12 +20,29 @@ struct timestamp uint8_t minute = 0; // minute [0-60] uint8_t second = 0; // second [0-60] uint32_t sub_second = 0; // 0.1 u_second [0 - 9,999,999] + + // Construct timestamp with all zero values + timestamp() = default; + + // Construct timestamp from values for each time component + timestamp(uint16_t yr, uint8_t mo, uint8_t dy, uint8_t h, uint8_t m, uint8_t s, uint32_t ss = 0); + + // Convert std::chrono::time_point to reinforcement_learning::timestamp + explicit timestamp(const std::chrono::time_point&); + + // Overload that typecasts duration to 100ns + template + explicit timestamp(const std::chrono::time_point& tp) + : timestamp(std::chrono::time_point_cast(tp)) + { + } + + // Convert to a std::chrono::time_point with 100ns resolution + std::chrono::time_point to_time_point() const; + friend std::ostream& operator<<(std::ostream& os, const timestamp& dt); }; -timestamp timestamp_from_chrono(const std::chrono::time_point&); -std::chrono::time_point chrono_from_timestamp(const timestamp&); - inline bool operator==(const timestamp& lhs, const timestamp& rhs) { return std::tie(lhs.year, lhs.month, lhs.day, lhs.hour, lhs.minute, lhs.second, lhs.sub_second) == diff --git a/rlclientlib/utility/vw_logger_adapter.cc b/rlclientlib/utility/vw_logger_adapter.cc new file mode 100644 index 000000000..f0eb04426 --- /dev/null +++ b/rlclientlib/utility/vw_logger_adapter.cc @@ -0,0 +1,37 @@ +#include "vw_logger_adapter.h" + +#include "trace_logger.h" + +static void vw_log_to_trace_logger(void* trace_logger, VW::io::log_level log_level, const std::string& msg) +{ + if (trace_logger == nullptr) { return; } + auto* i_trace_ptr = static_cast(trace_logger); + switch (log_level) + { + case VW::io::log_level::TRACE_LEVEL: + case VW::io::log_level::DEBUG_LEVEL: + TRACE_DEBUG(i_trace_ptr, msg); + break; + + case VW::io::log_level::INFO_LEVEL: + TRACE_INFO(i_trace_ptr, msg); + break; + + case VW::io::log_level::WARN_LEVEL: + TRACE_WARN(i_trace_ptr, msg); + break; + + case VW::io::log_level::ERROR_LEVEL: + case VW::io::log_level::CRITICAL_LEVEL: + TRACE_ERROR(i_trace_ptr, msg); + break; + + default: + break; + } +} + +VW::io::logger reinforcement_learning::utility::make_vw_trace_logger(i_trace* trace_logger) +{ + return VW::io::create_custom_sink_logger(trace_logger, vw_log_to_trace_logger); +} diff --git a/rlclientlib/utility/vw_logger_adapter.h b/rlclientlib/utility/vw_logger_adapter.h new file mode 100644 index 000000000..fe0a5fbf3 --- /dev/null +++ b/rlclientlib/utility/vw_logger_adapter.h @@ -0,0 +1,12 @@ +#include "trace_logger.h" +#include "vw/io/logger.h" + +#include + +namespace reinforcement_learning +{ +namespace utility +{ +VW::io::logger make_vw_trace_logger(i_trace* trace_logger); +} +} // namespace reinforcement_learning \ No newline at end of file From f0d551569c032f2da8e8cf37e6f5a1b0dde2f81b Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Thu, 2 Nov 2023 12:57:28 -0400 Subject: [PATCH 09/15] local loop factories --- rlclientlib/factory_resolver.cc | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rlclientlib/factory_resolver.cc b/rlclientlib/factory_resolver.cc index cb6880a24..d3f30806c 100644 --- a/rlclientlib/factory_resolver.cc +++ b/rlclientlib/factory_resolver.cc @@ -1,5 +1,6 @@ #include "factory_resolver.h" +#include "api_status.h" #include "constants.h" #include "err_constants.h" #include "logger/event_logger.h" @@ -8,6 +9,10 @@ #include "vw_model/pdf_model.h" #include "vw_model/vw_model.h" +#ifdef RL_BUILD_FEDERATION +# include "federation/local_loop_controller.h" +#endif + #ifdef USE_AZURE_FACTORIES # include "azure_factories.h" # include "model_mgmt/restapi_data_transport.h" @@ -115,6 +120,19 @@ int file_model_loader_create(std::unique_ptr& retval, const return error_code::success; } +#ifdef RL_BUILD_FEDERATION +int local_loop_controller_create(std::unique_ptr& retval, const u::configuration& config, + i_trace* trace_logger, api_status* status) +{ + TRACE_INFO(trace_logger, "Local loop controller i_data_transport created."); + std::unique_ptr output; + RETURN_IF_FAIL(local_loop_controller::create(output, config, trace_logger, status)); + retval = std::move(output); + return error_code::success; +} +#endif + + int null_time_provider_create( std::unique_ptr& retval, const u::configuration& config, i_trace* trace_logger, api_status* status) { @@ -140,6 +158,17 @@ void factory_initializer::register_default_factories() data_transport_factory.register_type(value::NO_MODEL_DATA, empty_data_transport_create); data_transport_factory.register_type(value::FILE_MODEL_DATA, file_model_loader_create); +#ifdef RL_BUILD_FEDERATION + data_transport_factory.register_type(value::LOCAL_LOOP_MODEL_DATA, local_loop_controller_create); +#else + data_transport_factory.register_type(value::LOCAL_LOOP_MODEL_DATA, + [](std::unique_ptr&, const u::configuration&, i_trace* trace_logger, api_status* status) + { + RETURN_ERROR_ARG(trace_logger, status, create_fn_exception, + "Cannot use LOCAL_LOOP_MODEL_DATA because rlclientlib was not compiled with federated learning enabled"); + }); +#endif + model_factory.register_type(value::VW, model_create); model_factory.register_type(value::PASSTHROUGH_PDF_MODEL, model_create); @@ -171,6 +200,15 @@ void factory_initializer::register_default_factories() const char* file_name = c.get(name::INTERACTION_FILE_NAME, "interaction.fb.data"); return file_sender_create(retval, c, file_name, cb, trace_logger, status); }); + + // Register a default factory for LOCAL_LOOP_SENDER that returns an error + sender_factory.register_type(value::LOCAL_LOOP_SENDER, + [](std::unique_ptr&, const u::configuration&, error_callback_fn*, i_trace* trace_logger, + api_status* status) + { + RETURN_ERROR_ARG(trace_logger, status, create_fn_exception, + "LOCAL_LOOP_SENDER must be used with model source set to LOCAL_LOOP_MODEL_DATA"); + }); } int null_tracer_create( From adec929a2379a6111922589a687bfbee049f3950 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Thu, 2 Nov 2023 13:17:20 -0400 Subject: [PATCH 10/15] live model update --- rlclientlib/live_model_impl.cc | 148 +++++++++++++++++++++++++++++---- rlclientlib/live_model_impl.h | 4 + 2 files changed, 137 insertions(+), 15 deletions(-) diff --git a/rlclientlib/live_model_impl.cc b/rlclientlib/live_model_impl.cc index 999667fa9..241844f51 100644 --- a/rlclientlib/live_model_impl.cc +++ b/rlclientlib/live_model_impl.cc @@ -17,10 +17,15 @@ #include "vw/explore/explore.h" #include "vw_model/safe_vw.h" +#ifdef RL_BUILD_FEDERATION +# include "federation/local_loop_controller.h" +#endif + #include #include #include #include +#include // Some namespace changes for more concise code namespace e = exploration; @@ -50,12 +55,57 @@ void default_error_callback(const api_status& status, void* watchdog_context) watchdog->set_unhandled_background_error(true); } +int live_model_impl::check_if_local_loop(bool& output, api_status* status) +{ + std::string model_src = _configuration.get(name::MODEL_SRC, value::get_default_data_transport()); + std::string interaction_sender = + _configuration.get(name::INTERACTION_SENDER_IMPLEMENTATION, value::get_default_interaction_sender()); + std::string observation_sender = + _configuration.get(name::OBSERVATION_SENDER_IMPLEMENTATION, value::get_default_observation_sender()); + + if (model_src != value::LOCAL_LOOP_MODEL_DATA && interaction_sender != value::LOCAL_LOOP_SENDER && + observation_sender != value::LOCAL_LOOP_SENDER) + { + // no local loop options used + output = false; + return error_code::success; + } + + if (model_src == value::LOCAL_LOOP_MODEL_DATA) + { + // (model_src == LOCAL_LOOP_MODEL_DATA) determines that local loop is used + // set default value of sender implementation to LOCAL_LOOP_SENDER + interaction_sender = _configuration.get(name::INTERACTION_SENDER_IMPLEMENTATION, value::LOCAL_LOOP_SENDER); + observation_sender = _configuration.get(name::OBSERVATION_SENDER_IMPLEMENTATION, value::LOCAL_LOOP_SENDER); + + // check that senders are set to allowed values here + // currently, only LOCAL_LOOP_SENDER is allowed + if (interaction_sender == value::LOCAL_LOOP_SENDER && observation_sender == value::LOCAL_LOOP_SENDER) + { + output = true; + return error_code::success; + } + } + + RETURN_ERROR_ARG(_trace_logger.get(), status, invalid_argument, + "Incompatible values for configuration options MODEL_SRC=", model_src, + " and INTERACTION_SENDER_IMPLEMENTATION=", interaction_sender, + " and OBSERVATION_SENDER_IMPLEMENTATION=", observation_sender); +} + int live_model_impl::init(api_status* status) { RETURN_IF_FAIL(init_trace(status)); RETURN_IF_FAIL(init_model(status)); - RETURN_IF_FAIL(init_model_mgmt(status)); - RETURN_IF_FAIL(init_loggers(status)); + + bool is_local_loop = false; + RETURN_IF_FAIL(check_if_local_loop(is_local_loop, status)); + if (is_local_loop) { RETURN_IF_FAIL(init_local_loop(status)); } + else + { + RETURN_IF_FAIL(init_model_mgmt(status)); + RETURN_IF_FAIL(init_loggers(status)); + } if (_protocol_version == 1) { @@ -506,7 +556,25 @@ int live_model_impl::init_loggers(api_status* status) RETURN_IF_FAIL(_sender_factory->create( ranking_data_sender, ranking_sender_impl, _configuration, &_error_cb, _trace_logger.get(), status)); RETURN_IF_FAIL(ranking_data_sender->init(_configuration, status)); + // Get the name of raw data (as opposed to message) sender for observations. + const auto* const outcome_sender_impl = + _configuration.get(name::OBSERVATION_SENDER_IMPLEMENTATION, value::get_default_observation_sender()); + std::unique_ptr outcome_sender; + // Use the name to create an instance of raw data sender for observations + _configuration.set(config_constants::CONFIG_SECTION, config_constants::OBSERVATION); + RETURN_IF_FAIL(_sender_factory->create( + outcome_sender, outcome_sender_impl, _configuration, &_error_cb, _trace_logger.get(), status)); + RETURN_IF_FAIL(outcome_sender->init(_configuration, status)); + + RETURN_IF_FAIL(init_loggers_common(std::move(ranking_data_sender), std::move(outcome_sender), status)); + return error_code::success; +} + +// Common part for both init_loggers and init_local_loop +int live_model_impl::init_loggers_common( + std::unique_ptr ranking_data_sender, std::unique_ptr outcome_sender, api_status* status) +{ // Create a message sender that will prepend the message with a preamble and send the raw data using the // factory created raw data sender std::unique_ptr ranking_msg_sender( @@ -534,17 +602,6 @@ int live_model_impl::init_loggers(api_status* status) std::move(ranking_msg_sender), _watchdog, std::move(ranking_time_provider), *_logger_extensions, &_error_cb)); RETURN_IF_FAIL(_interaction_logger->init(status)); - // Get the name of raw data (as opposed to message) sender for observations. - const auto* const outcome_sender_impl = - _configuration.get(name::OBSERVATION_SENDER_IMPLEMENTATION, value::get_default_observation_sender()); - std::unique_ptr outcome_sender; - - // Use the name to create an instance of raw data sender for observations - _configuration.set(config_constants::CONFIG_SECTION, config_constants::OBSERVATION); - RETURN_IF_FAIL(_sender_factory->create( - outcome_sender, outcome_sender_impl, _configuration, &_error_cb, _trace_logger.get(), status)); - RETURN_IF_FAIL(outcome_sender->init(_configuration, status)); - // Create a message sender that will prepend the message with a preamble and send the raw data using the // factory created raw data sender std::unique_ptr outcome_msg_sender(new l::preamble_message_sender(std::move(outcome_sender))); @@ -623,18 +680,79 @@ int live_model_impl::init_model_mgmt(api_status* status) { // Initialize transport for the model using transport factory const auto* const tranport_impl = _configuration.get(name::MODEL_SRC, value::get_default_data_transport()); - RETURN_IF_FAIL(_t_factory->create(_transport, tranport_impl, _configuration, status)); + std::unique_ptr ptransport; + RETURN_IF_FAIL(_t_factory->create(ptransport, tranport_impl, _configuration, status)); + // This class manages lifetime of transport + this->_transport = std::move(ptransport); if (_bg_model_proc) { // Initialize background process and start downloading models - _model_download.reset(new m::model_downloader(_transport.get(), &_data_cb, _trace_logger.get())); + this->_model_download.reset(new m::model_downloader(_transport.get(), &_data_cb, _trace_logger.get())); return _bg_model_proc->init(_model_download.get(), status); } return refresh_model(status); } +#ifdef RL_BUILD_FEDERATION +int live_model_impl::init_local_loop(api_status* status) +{ + std::string model_src = _configuration.get(name::MODEL_SRC, value::get_default_data_transport()); + + // This function should only be called when the configuration is set to use LOCAL_LOOP_MODEL_DATA + assert(model_src == value::LOCAL_LOOP_MODEL_DATA); + + // Creating i_data_transport with type LOCAL_LOOP_MODEL_DATA results in local_loop_controller + std::unique_ptr output; + RETURN_IF_FAIL(_t_factory->create(output, model_src, _configuration, _trace_logger.get(), status)); + std::unique_ptr llc(reinterpret_cast(output.release())); + + // Create senders with default sender implementation set to LOCAL_LOOP_SENDER + std::string interaction_sender_type = + _configuration.get(name::INTERACTION_SENDER_IMPLEMENTATION, value::LOCAL_LOOP_SENDER); + std::string observation_sender_type = + _configuration.get(name::INTERACTION_SENDER_IMPLEMENTATION, value::LOCAL_LOOP_SENDER); + + std::unique_ptr interaction_sender; + if (interaction_sender_type == value::LOCAL_LOOP_SENDER) { interaction_sender = llc->get_local_sender(); } + else + { + RETURN_IF_FAIL(_sender_factory->create( + interaction_sender, interaction_sender_type, _configuration, &_error_cb, _trace_logger.get(), status)); + RETURN_IF_FAIL(interaction_sender->init(_configuration, status)); + } + + std::unique_ptr observation_sender; + if (observation_sender_type == value::LOCAL_LOOP_SENDER) { observation_sender = llc->get_local_sender(); } + else + { + RETURN_IF_FAIL(_sender_factory->create( + observation_sender, observation_sender_type, _configuration, &_error_cb, _trace_logger.get(), status)); + RETURN_IF_FAIL(observation_sender->init(_configuration, status)); + } + + RETURN_IF_FAIL(init_loggers_common(std::move(interaction_sender), std::move(observation_sender), status)); + + // Set live_model_impl's data transport to local loop controller + _transport = std::move(llc); + + if (_bg_model_proc) + { + // Initialize background process and start downloading models + this->_model_download.reset(new m::model_downloader(_transport.get(), &_data_cb, _trace_logger.get())); + return _bg_model_proc->init(_model_download.get(), status); + } + return refresh_model(status); +} +#else +int live_model_impl::init_local_loop(api_status* status) +{ + RETURN_ERROR_ARG(_trace_logger.get(), status, invalid_argument, + "Cannot use LOCAL_LOOP_MODEL_DATA because library was compiled without support for federated learning"); +} +#endif + int live_model_impl::request_episodic_decision(const char* event_id, const char* previous_id, string_view context_json, unsigned int flags, ranking_response& resp, episode_state& episode, api_status* status) { diff --git a/rlclientlib/live_model_impl.h b/rlclientlib/live_model_impl.h index 8a5a0f237..601dcb268 100644 --- a/rlclientlib/live_model_impl.h +++ b/rlclientlib/live_model_impl.h @@ -81,7 +81,11 @@ class live_model_impl int init_model(api_status* status); int init_model_mgmt(api_status* status); int init_loggers(api_status* status); + int init_loggers_common(std::unique_ptr ranking_data_sender, + std::unique_ptr outcome_sender, api_status* status); int init_trace(api_status* status); + int init_local_loop(api_status* status); + int check_if_local_loop(bool& output, api_status* status); static void _handle_model_update(const model_management::model_data& data, live_model_impl* ctxt); void handle_model_update(const model_management::model_data& data); template From 74305c8064ef816fec5bf783c56f708bd452b42e Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Thu, 2 Nov 2023 13:52:07 -0400 Subject: [PATCH 11/15] unit test --- unit_test/CMakeLists.txt | 11 + unit_test/common_test_utils.h | 102 +++++- unit_test/data.h | 1 + unit_test/eud_test.cc | 33 ++ unit_test/local_client_test.cc | 103 ++++++ unit_test/local_loop_controller_test.cc | 207 +++++++++++ unit_test/local_loop_end_to_end.cc | 142 ++++++++ unit_test/sender_joined_log_provider_test.cc | 353 +++++++++++++++++++ unit_test/time_tests.cc | 16 +- unit_test/trainable_model_test.cc | 95 +++++ unit_test/watchdog_test.cc | 7 +- 11 files changed, 1057 insertions(+), 13 deletions(-) create mode 100644 unit_test/eud_test.cc create mode 100644 unit_test/local_client_test.cc create mode 100644 unit_test/local_loop_controller_test.cc create mode 100644 unit_test/local_loop_end_to_end.cc create mode 100644 unit_test/sender_joined_log_provider_test.cc create mode 100644 unit_test/trainable_model_test.cc diff --git a/unit_test/CMakeLists.txt b/unit_test/CMakeLists.txt index d857d5fd5..65be79147 100644 --- a/unit_test/CMakeLists.txt +++ b/unit_test/CMakeLists.txt @@ -48,6 +48,17 @@ if (vw_USE_AZURE_FACTORIES) ) endif() +if(RL_BUILD_FEDERATION) + list(APPEND TEST_SOURCES + eud_test.cc + local_client_test.cc + local_loop_controller_test.cc + local_loop_end_to_end.cc + sender_joined_log_provider_test.cc + trainable_model_test.cc + ) +endif() + # If compiling on windows add the stdafx file add_executable(rltest ${TEST_SOURCES}) diff --git a/unit_test/common_test_utils.h b/unit_test/common_test_utils.h index 8596f5a25..5088a261a 100644 --- a/unit_test/common_test_utils.h +++ b/unit_test/common_test_utils.h @@ -1,10 +1,24 @@ +#pragma once + #include +#include "model_mgmt.h" #include "rl_string_view.h" +#include "vw/config/options_cli.h" +#include "vw/core/array_parameters.h" +#include "vw/core/parse_primitives.h" +#include "vw/core/shared_data.h" +#include "vw/core/vw.h" +#include "vw/io/io_adapter.h" +#include #include -bool is_invoked_with(const std::string& arg) +namespace reinforcement_learning +{ +namespace test_utils +{ +inline bool is_invoked_with(const std::string& arg) { for (size_t i = 0; i < boost::unit_test::framework::master_test_suite().argc; i++) { @@ -15,4 +29,88 @@ bool is_invoked_with(const std::string& arg) } } return false; -} \ No newline at end of file +} + +inline std::unique_ptr create_vw(const std::string& command_line) +{ + auto opts = std::unique_ptr(new VW::config::options_cli(VW::split_command_line(command_line))); + return VW::initialize_experimental(std::move(opts)); +} + +inline std::unique_ptr create_vw( + const std::string& command_line, const model_management::model_data& data) +{ + auto opts = std::unique_ptr(new VW::config::options_cli(VW::split_command_line(command_line))); + auto data_reader = VW::io::create_buffer_view(data.data(), data.data_sz()); + return VW::initialize_experimental(std::move(opts), std::move(data_reader)); +} + +inline std::unique_ptr create_vw(const std::vector& command_line) +{ + auto opts = std::unique_ptr(new VW::config::options_cli(command_line)); + return VW::initialize_experimental(std::move(opts)); +} + +inline std::unique_ptr create_vw( + const std::vector& command_line, const model_management::model_data& data) +{ + auto opts = std::unique_ptr(new VW::config::options_cli(command_line)); + auto data_reader = VW::io::create_buffer_view(data.data(), data.data_sz()); + return VW::initialize_experimental(std::move(opts), std::move(data_reader)); +} + +inline model_management::model_data save_vw(VW::workspace& vw) +{ + io_buf io_buffer; + auto backing_buffer = std::make_shared>(); + io_buffer.add_file(VW::io::create_vector_writer(backing_buffer)); + VW::save_predictor(vw, io_buffer); + + model_management::model_data data; + auto* data_buffer = data.alloc(backing_buffer->size()); + std::memcpy(data_buffer, backing_buffer->data(), backing_buffer->size()); + + return data; +} + +inline void compare_vw(const VW::workspace& vw1, const VW::workspace& vw2) +{ + const auto sd1 = vw1.sd; + const auto sd2 = vw2.sd; + BOOST_CHECK_EQUAL(sd1->weighted_labeled_examples, sd2->weighted_labeled_examples); + BOOST_CHECK_EQUAL(sd1->weighted_labels, sd2->weighted_labels); + BOOST_CHECK_EQUAL(sd1->sum_loss, sd2->sum_loss); + BOOST_CHECK_EQUAL(sd1->total_features, sd2->total_features); + + // These will not necessarily be equal because in trainable_vw_model + // the binary parser will create and destroy a new example even when parsing is unsuccessful + // BOOST_CHECK_EQUAL(sd1->example_number, sd2->example_number); + // BOOST_CHECK_EQUAL(sd1->weighted_unlabeled_examples, sd2->weighted_unlabeled_examples); + + const auto& weights1 = vw1.weights; + const auto& weights2 = vw2.weights; + BOOST_CHECK_EQUAL(weights1.sparse, weights2.sparse); + + const float tolerance = 0.00001; + if (weights1.sparse) + { + auto& sw1 = weights1.sparse_weights; + auto& sw2 = weights2.sparse_weights; + for (auto it1 = sw1.cbegin(), it2 = sw2.cbegin(); it1 != sw1.cend() && it2 != sw2.cend(); ++it1, ++it2) + { + BOOST_CHECK_CLOSE(*it1, *it2, tolerance); + } + } + else + { + auto& dw1 = weights1.dense_weights; + auto& dw2 = weights2.dense_weights; + for (auto it1 = dw1.cbegin(), it2 = dw2.cbegin(); it1 != dw1.cend() && it2 != dw2.cend(); ++it1, ++it2) + { + BOOST_CHECK_CLOSE(*it1, *it2, tolerance); + } + } +} +} // namespace test_utils + +} // namespace reinforcement_learning \ No newline at end of file diff --git a/unit_test/data.h b/unit_test/data.h index c160d97a4..cb537d1d1 100644 --- a/unit_test/data.h +++ b/unit_test/data.h @@ -1,3 +1,4 @@ +#pragma once unsigned char regression_data_1_model[] = {0x06, 0x00, 0x00, 0x00, 0x38, 0x2e, 0x35, 0x2e, 0x30, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, diff --git a/unit_test/eud_test.cc b/unit_test/eud_test.cc new file mode 100644 index 000000000..ba00c031b --- /dev/null +++ b/unit_test/eud_test.cc @@ -0,0 +1,33 @@ +#include +#ifdef STAND_ALONE +# define BOOST_TEST_MODULE Main +#endif + +#include + +#include "federation/eud_utils.h" + +#include +#include + +using namespace reinforcement_learning; + +BOOST_AUTO_TEST_CASE(parse_eud_tests) +{ + std::chrono::seconds duration; + BOOST_CHECK_EQUAL(parse_eud("", duration, nullptr), error_code::invalid_argument); + BOOST_CHECK_EQUAL(parse_eud("::", duration, nullptr), error_code::invalid_argument); + BOOST_CHECK_EQUAL(parse_eud("a:b:cc", duration, nullptr), error_code::invalid_argument); + BOOST_CHECK_EQUAL(parse_eud("1a:1:11", duration, nullptr), error_code::invalid_argument); + BOOST_CHECK_EQUAL(parse_eud("a1:1:11", duration, nullptr), error_code::invalid_argument); + BOOST_CHECK_EQUAL(parse_eud("-1:1:11", duration, nullptr), error_code::invalid_argument); + + BOOST_CHECK_EQUAL(parse_eud("1:0:0", duration, nullptr), error_code::success); + BOOST_CHECK(duration == std::chrono::hours(1)); + BOOST_CHECK_EQUAL(parse_eud("1:25:0", duration, nullptr), error_code::success); + BOOST_CHECK(duration == std::chrono::hours(1) + std::chrono::minutes(25)); + BOOST_CHECK_EQUAL(parse_eud("1:25:1", duration, nullptr), error_code::success); + BOOST_CHECK(duration == std::chrono::hours(1) + std::chrono::minutes(25) + std::chrono::seconds(1)); + BOOST_CHECK_EQUAL(parse_eud("83:25:1", duration, nullptr), error_code::success); + BOOST_CHECK(duration == std::chrono::hours(83) + std::chrono::minutes(25) + std::chrono::seconds(1)); +} diff --git a/unit_test/local_client_test.cc b/unit_test/local_client_test.cc new file mode 100644 index 000000000..b99a46538 --- /dev/null +++ b/unit_test/local_client_test.cc @@ -0,0 +1,103 @@ +#ifdef STAND_ALONE +# define BOOST_TEST_MODULE Main +#endif + +#include + +#include "common_test_utils.h" +#include "err_constants.h" +#include "factory_resolver.h" +#include "federation/federated_client.h" +#include "federation/local_client.h" +#include "vw/core/merge.h" +#include "vw/core/parse_example.h" +#include "vw/core/parse_example_json.h" +#include "vw/core/vw.h" + +#include +#include + +using namespace reinforcement_learning; + +VW::multi_ex parse_json(VW::workspace& all, const std::string& line) +{ + VW::multi_ex examples; + examples.push_back(VW::new_unused_example(all)); + VW::example_factory_t ex_fac = [&all]() -> VW::example& { return *(VW::new_unused_example(all)); }; + VW::parsers::json::read_line_json(all, examples, (char*)line.c_str(), line.length(), ex_fac); + VW::setup_examples(all, examples); + return examples; +} + +BOOST_AUTO_TEST_CASE(get_model_twice_fails) +{ + utility::configuration config; + config.set("id", "test_app_id"); + std::unique_ptr client; + BOOST_CHECK_EQUAL(local_client::create(client, config, nullptr, nullptr), error_code::success); + model_management::model_data data; + bool model_received = false; + BOOST_CHECK_EQUAL(client->try_get_model("test_app_id", data, model_received), error_code::success); + BOOST_CHECK(data.data_sz() > 0); + BOOST_CHECK_EQUAL(model_received, true); + BOOST_CHECK_NE(client->try_get_model("test_app_id", data, model_received), error_code::success); +} + +BOOST_AUTO_TEST_CASE(send_delta_update) +{ + utility::configuration config; + config.set("id", "test_app_id"); + std::unique_ptr client; + BOOST_CHECK_EQUAL(local_client::create(client, config, nullptr, nullptr), error_code::success); + BOOST_CHECK_NE(client.get(), nullptr); + model_management::model_data data; + bool model_received = false; + BOOST_CHECK_EQUAL(client->try_get_model("test_app_id", data, model_received), error_code::success); + + auto original_workspace = test_utils::create_vw("", data); + auto workspace = test_utils::create_vw("", data); + std::string json_text = R"( + { + "s_": "1", + "s_": "2", + "_labelIndex": 0, + "_label_Action": 1, + "_label_Cost": 1, + "_label_Probability": 0.5, + "_multi": [ + { + "a_": "1", + "b_": "1", + "c_": "1" + }, + { + "a_": "2", + "b_": "2", + "c_": "2" + }, + { + "a_": "3", + "b_": "3", + "c_": "3" + } + ] + })"; + + auto examples = parse_json(*workspace, json_text); + workspace->learn(examples); + workspace->finish_example(examples); + + auto delta = *workspace - *original_workspace; + auto backing_buffer = std::make_shared>(); + auto writer = VW::io::create_vector_writer(backing_buffer); + delta.serialize(*writer); + BOOST_CHECK_EQUAL( + client->report_result(reinterpret_cast(backing_buffer->data()), backing_buffer->size()), + error_code::success); + BOOST_CHECK_NE( + client->report_result(reinterpret_cast(backing_buffer->data()), backing_buffer->size()), + error_code::success); + + model_received = false; + BOOST_CHECK_EQUAL(client->try_get_model("test_app_id", data, model_received), error_code::success); +} diff --git a/unit_test/local_loop_controller_test.cc b/unit_test/local_loop_controller_test.cc new file mode 100644 index 000000000..47d62792b --- /dev/null +++ b/unit_test/local_loop_controller_test.cc @@ -0,0 +1,207 @@ +#include + +#include "common_test_utils.h" +#include "configuration.h" +#include "constants.h" +#include "err_constants.h" +#include "federation/event_sink.h" +#include "federation/federated_client.h" +#include "federation/local_client.h" +#include "federation/local_loop_controller.h" +#include "federation/sender_joined_log_provider.h" +#include "vw/core/shared_data.h" +#include "vw/core/vw.h" + +using namespace reinforcement_learning; + +namespace +{ +// Wrapper around local_loop_controller to allow us to access member variables +class test_local_loop_controller : public local_loop_controller +{ +public: + test_local_loop_controller(std::string app_id, std::unique_ptr&& federated_client, + std::unique_ptr&& trainable_model, std::shared_ptr&& joiner, + std::shared_ptr&& event_sink) + : local_loop_controller(std::move(app_id), std::move(federated_client), std::move(trainable_model), + std::move(joiner), std::move(event_sink)) + { + } + + virtual ~test_local_loop_controller() = default; + + i_federated_client* get_client() { return _federated_client.get(); } + trainable_vw_model* get_model() { return _trainable_model.get(); } + i_joined_log_provider* get_joiner() { return _joiner.get(); } + i_event_sink* get_event_sink() { return _event_sink.get(); } +}; + +// Mock version that simply stores and reads data +class mock_federated_client : public i_federated_client +{ +public: + virtual int try_get_model(const std::string& app_id, model_management::model_data& data, bool& model_received, + api_status* status = nullptr) override + { + if (_need_to_report_result) return -1; + if (_has_data) + { + data = std::move(_data); + _has_data = false; + model_received = true; + _need_to_report_result = true; + } + else + model_received = false; + return error_code::success; + } + + virtual int report_result(const uint8_t* payload, size_t size, api_status* status = nullptr) override + { + if (!_need_to_report_result) return -1; + _result.clear(); + _result.reserve(size); + _result.insert(_result.begin(), payload, payload + size); + _has_result = true; + _need_to_report_result = false; + return error_code::success; + } + + mock_federated_client() = default; + virtual ~mock_federated_client() = default; + + void load_model_data(model_management::model_data data) + { + _data = std::move(data); + _has_data = true; + } + + std::vector get_result() + { + if (_has_result) + { + _has_result = false; + return std::move(_result); + } + return std::vector(); + } + + model_management::model_data _data; + bool _has_data = false; + std::vector _result; + bool _has_result = false; + bool _need_to_report_result = false; +}; + +// Mock version that simply stores and reads data +class mock_event_sink : public i_event_sink +{ +public: + using buffer = std::shared_ptr; + + virtual int receive_events(const buffer& data, api_status* status = nullptr) override + { + _data = data; + return error_code::success; + } + + buffer get_latest_event() { return _data; } + + virtual ~mock_event_sink() = default; + + buffer _data; +}; + +utility::configuration get_test_config() +{ + utility::configuration config; + config.set(name::MODEL_VW_INITIAL_COMMAND_LINE, "--quiet --preserve_performance_counters"); + config.set(name::PROTOCOL_VERSION, "2"); + config.set(name::JOINER_EUD_DURATION, "0:0:0"); + return config; +} + +std::unique_ptr create_test_local_loop_controller(utility::configuration config) +{ + std::unique_ptr trainable_model; + std::unique_ptr sender_joiner; + BOOST_CHECK_EQUAL(trainable_vw_model::create(trainable_model, config), error_code::success); + BOOST_CHECK_EQUAL(sender_joined_log_provider::create(sender_joiner, config), error_code::success); + + std::shared_ptr joiner = std::move(sender_joiner); + std::shared_ptr event_sink(new mock_event_sink()); + std::unique_ptr federated_client(new mock_federated_client()); + + return std::unique_ptr(new test_local_loop_controller("test_app_id", + std::move(federated_client), std::move(trainable_model), std::move(joiner), std::move(event_sink))); +} +} // namespace + +BOOST_AUTO_TEST_CASE(sender_factory_test) +{ + // create the local_loop_controller + auto config = get_test_config(); + auto test_llc = create_test_local_loop_controller(config); + + // create a sender and send some data + std::unique_ptr sender = test_llc->get_local_sender(); + const char* test_data = "Testing the sender..."; + auto test_data_len = std::char_traits::length(test_data); + + std::shared_ptr buffer_in = VW::make_unique(test_data_len); + std::memcpy(buffer_in->raw_begin(), test_data, test_data_len); + sender->send(buffer_in); + + // get the data out of event sink + auto event_sink_out = dynamic_cast(test_llc.get())->get_event_sink(); + BOOST_CHECK_NE(event_sink_out, nullptr); + auto buffer_out = dynamic_cast(event_sink_out)->get_latest_event(); + BOOST_CHECK_NE(buffer_out, nullptr); + BOOST_CHECK_EQUAL(std::memcmp(buffer_out->raw_begin(), test_data, test_data_len), 0); +} + +BOOST_AUTO_TEST_CASE(update_get_model_data) +{ + // create the local_loop_controller + auto config = get_test_config(); + auto llc = create_test_local_loop_controller(config); + auto test_llc = dynamic_cast(llc.get()); + BOOST_CHECK_NE(test_llc, nullptr); + auto mock_client = dynamic_cast(test_llc->get_client()); + BOOST_CHECK_NE(mock_client, nullptr); + + // create a model and train on an example + const std::string command_line = config.get(name::MODEL_VW_INITIAL_COMMAND_LINE, ""); + auto vw = test_utils::create_vw(command_line); + auto ex = VW::read_example(*vw, "1 | a"); + vw->learn(*ex); + vw->finish_example(*ex); + + // this should update the internal model + // check that data retrieved is the same as data provided + model_management::model_data serialized_vw = test_utils::save_vw(*vw); + mock_client->load_model_data(serialized_vw); + model_management::model_data data_out; + BOOST_CHECK_EQUAL(llc->get_data(data_out), error_code::success); + test_utils::compare_vw(*vw, *test_utils::create_vw(command_line, data_out)); + + // this should generate a model delta + // check that model delta has nonzero size + // check that model data has not changed + BOOST_CHECK_EQUAL(llc->get_data(data_out), error_code::success); + auto serialized_delta = mock_client->get_result(); + BOOST_CHECK_NE(serialized_delta.size(), 0); + test_utils::compare_vw(*vw, *test_utils::create_vw(command_line, data_out)); + + // this should do nothing since mock_client has no new data + // check that model data has not changed + BOOST_CHECK_EQUAL(llc->get_data(data_out), error_code::success); + test_utils::compare_vw(*vw, *test_utils::create_vw(command_line, data_out)); + + // delta should do nothing since we didn't train on any new examples + auto delta_reader = + VW::io::create_buffer_view(reinterpret_cast(serialized_delta.data()), serialized_delta.size()); + auto delta = VW::model_delta::deserialize(*delta_reader); + auto vw_new = *vw + *delta; + test_utils::compare_vw(*vw, *vw_new); +} diff --git a/unit_test/local_loop_end_to_end.cc b/unit_test/local_loop_end_to_end.cc new file mode 100644 index 000000000..edf48afc1 --- /dev/null +++ b/unit_test/local_loop_end_to_end.cc @@ -0,0 +1,142 @@ +#include + +#include "common_test_utils.h" +#include "constants.h" +#include "federation/local_loop_controller.h" +#include "live_model.h" +#include "ranking_response.h" + +#include +#include + +using namespace reinforcement_learning; + +namespace +{ +void pick_context_and_desired_action(std::string& context, size_t& action) +{ + int random = std::rand() % 4; + switch (random) + { + case 0: + context = + R"({ "shared":{ "name":"Anna", "time":"Morning" }, "_multi":[{ "TAction":{"article":"Sports"} }, { "TAction":{"article":"Politics"} }, { "TAction":{"article":"Food"} }] })"; + action = 0; + break; + + case 1: + context = + R"({ "shared":{ "name":"Tom", "time":"Morning" }, "_multi":[{ "TAction":{"article":"Sports"} }, { "TAction":{"article":"Politics"} }, { "TAction":{"article":"Food"} }] })"; + action = 1; + break; + + case 2: + context = + R"({ "shared":{ "name":"Anna", "time":"Afternoon" }, "_multi":[{ "TAction":{"article":"Sports"} }, { "TAction":{"article":"Politics"} }, { "TAction":{"article":"Food"} }] })"; + action = 2; + break; + + case 3: + context = + R"({ "shared":{ "name":"Tom", "time":"Afternoon" }, "_multi":[{ "TAction":{"article":"Sports"} }, { "TAction":{"article":"Politics"} }, { "TAction":{"article":"Food"} }] })"; + action = 2; + break; + } +} + +float run_simulation(live_model& model, api_status& status, int iterations) +{ + float reward = 0.f; + for (int i = 0; i < iterations; i++) + { + std::string context; + size_t desired_action; + pick_context_and_desired_action(context, desired_action); + + ranking_response response; + model.choose_rank(context, response, &status); + BOOST_TEST(status.get_error_code() == error_code::success, status.get_error_msg()); + + size_t chosen_action; + response.get_chosen_action_id(chosen_action, &status); + BOOST_TEST(status.get_error_code() == error_code::success, status.get_error_msg()); + + float outcome = chosen_action == desired_action ? 1.f : 0.f; + reward += outcome; + model.report_outcome(response.get_event_id(), outcome, &status); + BOOST_TEST(status.get_error_code() == error_code::success, status.get_error_msg()); + } + return reward; +} + +utility::configuration get_test_config() +{ + utility::configuration config; + + // -q :: is necessary here + config.set(name::MODEL_VW_INITIAL_COMMAND_LINE, + "--cb_explore_adf --json --quiet --epsilon 0.0 --preserve_performance_counters -q ::"); + config.set(name::PROTOCOL_VERSION, "2"); + config.set(name::MODEL_SRC, value::LOCAL_LOOP_MODEL_DATA); + config.set(name::INTERACTION_SENDER_IMPLEMENTATION, value::LOCAL_LOOP_SENDER); + config.set(name::OBSERVATION_SENDER_IMPLEMENTATION, value::LOCAL_LOOP_SENDER); + config.set(name::JOINER_EUD_DURATION, "0:0:1"); + config.set(name::JOINER_PROBLEM_TYPE, value::PROBLEM_TYPE_CB); + config.set(name::JOINER_REWARD_FUNCTION, value::REWARD_FUNCTION_EARLIEST); + config.set(name::JOINER_LEARNING_MODE, value::LEARNING_MODE_ONLINE); + config.set(name::MODEL_BACKGROUND_REFRESH, "false"); + config.set(name::TIME_PROVIDER_IMPLEMENTATION, value::CLOCK_TIME_PROVIDER); + + return config; +} +} // namespace + +BOOST_AUTO_TEST_CASE(local_loop_end_to_end_test) +{ + auto config = get_test_config(); + + // create a custom data_transport_factory_t that saves a pointer + // to the local_loop_controller that was created + local_loop_controller* test_local_loop_controller = nullptr; + data_transport_factory_t test_data_transport_factory; + test_data_transport_factory.register_type(value::LOCAL_LOOP_MODEL_DATA, + [&](std::unique_ptr& retval, const utility::configuration& cfg, + i_trace* trace_logger, api_status* status) + { + std::unique_ptr output; + RETURN_IF_FAIL(local_loop_controller::create(output, cfg, trace_logger, status)); + test_local_loop_controller = output.get(); + retval = std::move(output); + return error_code::success; + }); + + // initialize live_model with the custom data transport factory + api_status status; + live_model model( + config, nullptr, nullptr, &reinforcement_learning::trace_logger_factory, &test_data_transport_factory); + model.init(&status); + BOOST_TEST(status.get_error_code() == error_code::success, status.get_error_msg()); + BOOST_CHECK_NE(test_local_loop_controller, nullptr); + + // do some inference calls and report the outcome + constexpr int iterations = 100; + auto reward_before_update = run_simulation(model, status, iterations); + + // wait past eud time and update model + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); + model.refresh_model(&status); + BOOST_TEST(status.get_error_code() == error_code::success, status.get_error_msg()); + + // check that updated model has learned from previous outcomes + model_management::model_data model_data; + test_local_loop_controller->get_data(model_data, &status); + BOOST_TEST(status.get_error_code() == error_code::success, status.get_error_msg()); + auto vw = test_utils::create_vw(config.get(name::MODEL_VW_INITIAL_COMMAND_LINE, nullptr), model_data); + BOOST_CHECK_EQUAL(vw->sd->weighted_labeled_examples, iterations); + + // check that updated model now has better statistical performance + auto reward_after_update = run_simulation(model, status, iterations); + BOOST_CHECK_GT(reward_after_update, reward_before_update); + // std::cerr << "Reward before training: " << reward_before_update << std::endl; + // std::cerr << "Reward after training: " << reward_after_update << std::endl; +} diff --git a/unit_test/sender_joined_log_provider_test.cc b/unit_test/sender_joined_log_provider_test.cc new file mode 100644 index 000000000..5582c8479 --- /dev/null +++ b/unit_test/sender_joined_log_provider_test.cc @@ -0,0 +1,353 @@ +#include + +#include "configuration.h" +#include "constants.h" +#include "data_buffer.h" +#include "err_constants.h" +#include "federation/sender_joined_log_provider.h" +#include "generated/v2/Event_generated.h" +#include "generated/v2/FileFormat_generated.h" +#include "generated/v2/Metadata_generated.h" +#include "logger/message_type.h" +#include "logger/preamble.h" +#include "time_helper.h" +#include "vw/core/io_buf.h" +#include "vw/io/io_adapter.h" + +#include + +#include +#include +#include +#include + +using namespace reinforcement_learning; +namespace fbv2 = reinforcement_learning::messages::flatbuff::v2; + +namespace +{ +constexpr size_t BINARY_PARSER_VERSION = 1; +constexpr uint32_t MSG_TYPE_FILEMAGIC = 0x42465756; //'VWFB' +constexpr uint32_t MSG_TYPE_HEADER = 0x55555555; +constexpr uint32_t MSG_TYPE_REGULAR = 0xFFFFFFFF; +constexpr uint32_t MSG_TYPE_CHECKPOINT = 0x11111111; +constexpr uint32_t MSG_TYPE_EOF = 0xAAAAAAAA; + +// Object containing event metadata and string payload +// with helper functions to convert to/from Event flatbuffers +struct test_event +{ + std::string _event_id; + timestamp _time; + fbv2::PayloadType _payload_type; + std::string _payload; + + test_event(std::string event_id, timestamp time, fbv2::PayloadType payload_type, std::string payload) + : _event_id(std::move(event_id)), _time(time), _payload_type(payload_type), _payload(std::move(payload)) + { + } + + test_event(const fbv2::Event* buffer) + { + auto meta = buffer->meta(); + auto ts = meta->client_time_utc(); + auto payload = buffer->payload(); + _event_id = flatbuffers::GetString(meta->id()); + _time.year = ts->year(); + _time.day = ts->day(); + _time.month = ts->month(); + _time.hour = ts->hour(); + _time.minute = ts->minute(); + _time.second = ts->second(); + _time.sub_second = ts->subsecond(); + _payload_type = meta->payload_type(); + _payload = std::string(payload->begin(), payload->end()); + } + + flatbuffers::DetachedBuffer to_flatbuffer() + { + flatbuffers::FlatBufferBuilder event_builder; + auto ts = + fbv2::TimeStamp(_time.year, _time.month, _time.day, _time.hour, _time.minute, _time.second, _time.sub_second); + auto metadata = fbv2::CreateMetadataDirect(event_builder, _event_id.c_str(), &ts, nullptr, _payload_type); + auto payload_bytes = std::vector(_payload.begin(), _payload.end()); + auto event = fbv2::CreateEventDirect(event_builder, metadata, &payload_bytes); + event_builder.Finish(event); + return event_builder.Release(); + } +}; // struct test_event + +inline bool operator<(const test_event& te1, const test_event& te2) +{ + return std::tie(te1._event_id, te1._time, te1._payload_type, te1._payload) < + std::tie(te2._event_id, te2._time, te2._payload_type, te2._payload); +} + +inline bool operator==(const test_event& te1, const test_event& te2) +{ + return std::tie(te1._event_id, te1._time, te1._payload_type, te1._payload) == + std::tie(te2._event_id, te2._time, te2._payload_type, te2._payload); +} + +// A group of unique test_event objects with the same event id +struct test_event_batch +{ + std::set _batch; + + bool add_event(test_event evt) + { + auto result = _batch.insert(std::move(evt)); + return std::get<1>(result); + } + + bool verify() + { + // check that batch is non-empty and all events have the same id + if (_batch.empty()) return false; + auto first_id = _batch.begin()->_event_id; + for (auto&& evt : _batch) + { + if (evt._event_id != first_id) return false; + } + return true; + } + + bool operator==(const std::set& other) { return _batch == other; } +}; // struct test_event_batch + +std::shared_ptr create_message(test_event t_event) +{ + // create the Event flatbuffer + auto event_buffer = t_event.to_flatbuffer(); + auto serialized_buffer = std::vector(event_buffer.data(), event_buffer.data() + event_buffer.size()); + + // create the EventBatch flatbuffer + flatbuffers::FlatBufferBuilder batch_builder; + auto serialized_event = fbv2::CreateSerializedEventDirect(batch_builder, &serialized_buffer); + std::vector serialized_event_vector = {serialized_event}; + auto batch_metadata = fbv2::CreateBatchMetadataDirect(batch_builder, "IDENTITY"); + auto event_batch = fbv2::CreateEventBatchDirect(batch_builder, &serialized_event_vector, batch_metadata); + batch_builder.Finish(event_batch); + auto event_batch_buffer = batch_builder.Release(); + + // create preamble and put the flatbuffer into utility::data_buffer + logger::preamble pre; + pre.reserved = 0; + pre.version = 0; + pre.msg_type = logger::message_type::fb_generic_event_collection; + pre.msg_size = static_cast(event_batch_buffer.size()); + auto data_buffer = std::make_shared(event_batch_buffer.size()); + BOOST_TEST(pre.write_to_bytes(data_buffer->preamble_begin(), data_buffer->preamble_size())); + std::memcpy(data_buffer->body_begin(), event_batch_buffer.data(), event_batch_buffer.size()); + data_buffer->set_body_endoffset(data_buffer->get_body_beginoffset() + event_batch_buffer.size()); + return data_buffer; +} + +bool read_uint32(io_buf& buffer, uint32_t& output) +{ + char* read_ptr = nullptr; + auto len = buffer.buf_read(read_ptr, sizeof(uint32_t)); + if (len != sizeof(uint32_t) || read_ptr == nullptr) return false; + output = *reinterpret_cast(read_ptr); + return true; +} + +bool read_message(io_buf& buffer, uint32_t& message_type_out, std::vector& payload_out) +{ + message_type_out = 0; + payload_out.clear(); + + if (!read_uint32(buffer, message_type_out)) + { + // end of file + return false; + } + + uint32_t size = 0; + BOOST_TEST(read_uint32(buffer, size), "could not read payload size"); + + if (message_type_out == MSG_TYPE_FILEMAGIC) + { + // for file magic message, size is actually version number + BOOST_CHECK_EQUAL(size, BINARY_PARSER_VERSION); + return true; + } + + // if not file magic, must be another valid message type + BOOST_CHECK(message_type_out == MSG_TYPE_HEADER || message_type_out == MSG_TYPE_REGULAR || + message_type_out == MSG_TYPE_CHECKPOINT || message_type_out == MSG_TYPE_EOF); + + // all other message types have a real payload + char* read_ptr = nullptr; + auto actual_bytes_read = buffer.buf_read(read_ptr, size); + BOOST_CHECK_EQUAL(actual_bytes_read, size); + payload_out.insert(payload_out.begin(), read_ptr, read_ptr + size); + + // read padding bytes + auto padding_size = size % 8; + actual_bytes_read = buffer.buf_read(read_ptr, padding_size); + BOOST_CHECK_EQUAL(actual_bytes_read, padding_size); + return true; +} + +std::vector parse_joined_log(std::unique_ptr&& joined_log) +{ + io_buf buffer; + buffer.add_file(std::move(joined_log)); + + // must begin with file magic message + uint32_t message_type; + std::vector payload; + BOOST_TEST(read_message(buffer, message_type, payload), "could not read file magic message at start of binary log"); + BOOST_CHECK_EQUAL(message_type, MSG_TYPE_FILEMAGIC); + BOOST_TEST(payload.empty(), "file magic message has non-empty payload"); + + std::vector output; + while (read_message(buffer, message_type, payload)) + { + if (message_type == MSG_TYPE_REGULAR) + { + auto joined_payload = flatbuffers::GetRoot(payload.data()); + auto joined_payload_verifier = flatbuffers::Verifier(payload.data(), payload.size()); + BOOST_TEST(joined_payload->Verify(joined_payload_verifier), "verification failed on JoinedPayload flatbuffer"); + + test_event_batch batch; + auto joined_payload_events = joined_payload->events(); + BOOST_CHECK_NE(joined_payload_events, nullptr); + + for (auto joined_event : *joined_payload_events) + { + BOOST_CHECK_NE(joined_event, nullptr); + BOOST_CHECK_NE(joined_event->event(), nullptr); + auto event_fb = flatbuffers::GetRoot(joined_event->event()->data()); + auto event_verifier = flatbuffers::Verifier(joined_event->event()->data(), joined_event->event()->size()); + BOOST_TEST(event_fb->Verify(event_verifier), "verification failed on Event flatbuffer"); + + BOOST_CHECK_NE(event_fb->payload(), nullptr); + BOOST_CHECK_NE(event_fb->meta(), nullptr); + BOOST_CHECK_NE(event_fb->meta()->id(), nullptr); + BOOST_CHECK_NE(event_fb->meta()->client_time_utc(), nullptr); + BOOST_TEST(batch.add_event(test_event(event_fb)), "tried to insert duplicate event into event batch"); + } + + BOOST_TEST(batch.verify(), "event batch was empty or has bad event id"); + output.push_back(std::move(batch)); + } + + if (message_type == MSG_TYPE_EOF) { break; } + } + return output; +} + +std::unique_ptr create_test_object() +{ + utility::configuration config; + config.set(name::MODEL_VW_INITIAL_COMMAND_LINE, "--quiet --preserve_performance_counters"); + config.set(name::PROTOCOL_VERSION, "2"); + config.set(name::JOINER_EUD_DURATION, "0:0:10"); // EUD set to 10 seconds for all tests here + + std::unique_ptr sjlp; + BOOST_CHECK_EQUAL(sender_joined_log_provider::create(sjlp, config), error_code::success); + return sjlp; +} + +timestamp get_time(int seconds_from_now = 0) +{ + auto now = std::chrono::system_clock::now(); + auto time = now + std::chrono::seconds(seconds_from_now); + return timestamp(time); +} + +} // namespace + +BOOST_AUTO_TEST_CASE(empty_join) +{ + auto sjlp = create_test_object(); + std::unique_ptr output; + BOOST_CHECK_EQUAL(sjlp->invoke_join(output), error_code::success); + + auto result = parse_joined_log(std::move(output)); + BOOST_CHECK_EQUAL(result.size(), 0); +} + +BOOST_AUTO_TEST_CASE(one_interaction_before_eud) +{ + auto sjlp = create_test_object(); + test_event evt("id", get_time(0), fbv2::PayloadType_CB, "test payload"); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt)), error_code::success); + + std::unique_ptr output; + BOOST_CHECK_EQUAL(sjlp->invoke_join(output), error_code::success); + + auto result = parse_joined_log(std::move(output)); + BOOST_CHECK_EQUAL(result.size(), 0); +} + +BOOST_AUTO_TEST_CASE(one_interaction_after_eud) +{ + auto sjlp = create_test_object(); + test_event evt("id", get_time(-99), fbv2::PayloadType_CB, "test payload"); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt)), error_code::success); + + std::unique_ptr output; + BOOST_CHECK_EQUAL(sjlp->invoke_join(output), error_code::success); + + auto result = parse_joined_log(std::move(output)); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK(result[0] == (std::set{evt})); +} + +BOOST_AUTO_TEST_CASE(one_interaction_with_observations) +{ + auto sjlp = create_test_object(); + test_event evt1("id", get_time(-20), fbv2::PayloadType_CB, "test payload"); + test_event evt2("id", get_time(-19), fbv2::PayloadType_Outcome, "observation 1"); + test_event evt3("id", get_time(-15), fbv2::PayloadType_Outcome, "observation 2"); + test_event evt4("id", get_time(-5), fbv2::PayloadType_Outcome, "this observation is past eud"); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt1)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt2)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt3)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt4)), error_code::success); + + std::unique_ptr output; + BOOST_CHECK_EQUAL(sjlp->invoke_join(output), error_code::success); + + auto result = parse_joined_log(std::move(output)); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK(result[0] == (std::set{evt1, evt2, evt3})); +} + +BOOST_AUTO_TEST_CASE(multiple_interactions_and_observations) +{ + auto sjlp = create_test_object(); + test_event evt1("id_0", get_time(-20), fbv2::PayloadType_CB, "test payload"); + test_event evt2("id_0", get_time(-19), fbv2::PayloadType_Outcome, "observation 1"); + test_event evt3("id_0", get_time(-15), fbv2::PayloadType_Outcome, "observation 2"); + test_event evt4("id_0", get_time(-5), fbv2::PayloadType_Outcome, "this observation is past eud"); + test_event evt5("id_1", get_time(-80), fbv2::PayloadType_CB, "test payload"); + test_event evt6("id_2", get_time(-50), fbv2::PayloadType_CB, "test payload"); + test_event evt7("id_2", get_time(-49), fbv2::PayloadType_Outcome, "observation 1"); + test_event evt8("id_3", get_time(-1), fbv2::PayloadType_CB, "this event is before eud"); + test_event evt9("id_4", get_time(-15), fbv2::PayloadType_CB, "test payload"); + + // add events in order of time + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt5)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt6)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt7)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt1)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt2)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt9)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt3)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt4)), error_code::success); + BOOST_CHECK_EQUAL(sjlp->receive_events(create_message(evt8)), error_code::success); + + std::unique_ptr output; + BOOST_CHECK_EQUAL(sjlp->invoke_join(output), error_code::success); + + auto result = parse_joined_log(std::move(output)); + BOOST_CHECK_EQUAL(result.size(), 4); + BOOST_CHECK(result[0] == (std::set{evt5})); + BOOST_CHECK(result[1] == (std::set{evt6, evt7})); + BOOST_CHECK(result[2] == (std::set{evt1, evt2, evt3})); + BOOST_CHECK(result[3] == (std::set{evt9})); +} diff --git a/unit_test/time_tests.cc b/unit_test/time_tests.cc index 58b996e54..963702ece 100644 --- a/unit_test/time_tests.cc +++ b/unit_test/time_tests.cc @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(time_round_trip) { r::clock_time_provider ctp; auto now = ctp.gmt_now(); - auto roundtripped = r::timestamp_from_chrono(r::chrono_from_timestamp(now)); + auto roundtripped = r::timestamp(now.to_time_point()); BOOST_CHECK_EQUAL(now, roundtripped); } @@ -41,13 +41,13 @@ BOOST_AUTO_TEST_CASE(time_ordering) { r::clock_time_provider ctp; auto now = ctp.gmt_now(); - BOOST_CHECK(r::timestamp_from_chrono(std::chrono::system_clock::now() - std::chrono::seconds(5)) < now); - BOOST_CHECK(r::timestamp_from_chrono(std::chrono::system_clock::now() - std::chrono::minutes(5)) < now); - BOOST_CHECK(r::timestamp_from_chrono(std::chrono::system_clock::now() - std::chrono::hours(5)) < now); - BOOST_CHECK(r::timestamp_from_chrono(std::chrono::system_clock::now() - std::chrono::hours(500)) < now); - BOOST_CHECK(r::timestamp_from_chrono(std::chrono::system_clock::now() - date::days(1)) < now); - BOOST_CHECK(r::timestamp_from_chrono(std::chrono::system_clock::now() - date::months(1)) < now); - BOOST_CHECK(r::timestamp_from_chrono(std::chrono::system_clock::now() - date::years(1)) < now); + BOOST_CHECK(r::timestamp(std::chrono::system_clock::now() - std::chrono::seconds(5)) < now); + BOOST_CHECK(r::timestamp(std::chrono::system_clock::now() - std::chrono::minutes(5)) < now); + BOOST_CHECK(r::timestamp(std::chrono::system_clock::now() - std::chrono::hours(5)) < now); + BOOST_CHECK(r::timestamp(std::chrono::system_clock::now() - std::chrono::hours(500)) < now); + BOOST_CHECK(r::timestamp(std::chrono::system_clock::now() - date::days(1)) < now); + BOOST_CHECK(r::timestamp(std::chrono::system_clock::now() - date::months(1)) < now); + BOOST_CHECK(r::timestamp(std::chrono::system_clock::now() - date::years(1)) < now); } // BOOST_AUTO_TEST_CASE(time_loop) { diff --git a/unit_test/trainable_model_test.cc b/unit_test/trainable_model_test.cc new file mode 100644 index 000000000..a5b9b3baf --- /dev/null +++ b/unit_test/trainable_model_test.cc @@ -0,0 +1,95 @@ +#include + +#include "common_test_utils.h" +#include "constants.h" +#include "err_constants.h" +#include "federation/vw_trainable_model.h" +#include "vw/core/shared_data.h" +#include "vw/core/vw.h" + +#include + +using namespace reinforcement_learning; + +void setup_config(utility::configuration& config) +{ + config.set(name::PROTOCOL_VERSION, "2"); + config.set(name::MODEL_VW_INITIAL_COMMAND_LINE, "--quiet --preserve_performance_counters"); + config.set(name::JOINER_PROBLEM_TYPE, value::PROBLEM_TYPE_UNKNOWN); + config.set(name::JOINER_LEARNING_MODE, value::LEARNING_MODE_ONLINE); + config.set(name::JOINER_REWARD_FUNCTION, value::REWARD_FUNCTION_EARLIEST); +} + +BOOST_AUTO_TEST_CASE(trainable_model_set_get_data) +{ + utility::configuration config; + setup_config(config); + const std::string command_line = config.get(name::MODEL_VW_INITIAL_COMMAND_LINE, ""); + auto vw = test_utils::create_vw(command_line); + + // learn on one example + auto ex = VW::read_example(*vw, "1 | a"); + vw->learn(*ex); + vw->finish_example(*ex); + const auto example_count = vw->sd->weighted_labeled_examples; + BOOST_CHECK_EQUAL(example_count, 1.f); + + // put the workspace into trainable_vw_model + std::unique_ptr model; + BOOST_CHECK_EQUAL(trainable_vw_model::create(model, config), error_code::success); + BOOST_CHECK_EQUAL(model->set_model(std::move(vw)), error_code::success); + + // get data out and check that it's equal + model_management::model_data data_out; + BOOST_CHECK_EQUAL(model->get_data(data_out), error_code::success); + auto vw_out = test_utils::create_vw(command_line, data_out); + BOOST_CHECK_EQUAL(vw_out->sd->weighted_labeled_examples, example_count); +} + +BOOST_AUTO_TEST_CASE(trainable_model_learn_and_create_delta) +{ + const std::string command_line = "--quiet --preserve_performance_counters"; + + // create 2 copies of the base VW workspace + auto vw1 = test_utils::create_vw(command_line); + auto vw2 = test_utils::create_vw(command_line); + + // learn on one example + VW::example* ex = VW::read_example(*vw1, "1 | a"); + vw1->learn(*ex); + vw1->finish_example(*ex); + ex = VW::read_example(*vw2, "1 | a"); + vw2->learn(*ex); + vw2->finish_example(*ex); + + // put the workspace into trainable_vw_model + std::unique_ptr trainable_model; + utility::configuration config; + config.set(name::PROTOCOL_VERSION, "2"); + config.set(name::MODEL_VW_INITIAL_COMMAND_LINE, command_line.c_str()); + BOOST_CHECK_EQUAL(trainable_vw_model::create(trainable_model, config), error_code::success); + BOOST_CHECK_EQUAL(trainable_model->set_model(std::move(vw1)), error_code::success); + + // train the trainable_vw_model on another example + auto vw3 = test_utils::create_vw(command_line); + std::vector examples; + examples.push_back(VW::read_example(*vw3, "1 | b")); + BOOST_CHECK_EQUAL(trainable_model->learn(*vw3, examples), error_code::success); + vw3->finish_example(*examples.back()); + + // get data in trainable model + model_management::model_data data_out; + BOOST_CHECK_EQUAL(trainable_model->get_data(data_out), error_code::success); + auto vw1_updated = test_utils::create_vw(command_line, data_out); + + // get model delta and update vw2 workspace + VW::model_delta delta(nullptr); + BOOST_CHECK_EQUAL(trainable_model->get_model_delta(delta), error_code::success); + auto vw2_updated = *vw2 + delta; + VW::workspace* delta_ws = delta.unsafe_get_workspace_ptr(); + BOOST_CHECK_EQUAL(delta_ws->sd->weighted_labeled_examples, 1.f); + + // check that results are same + BOOST_CHECK_EQUAL(vw1_updated->sd->weighted_labeled_examples, 2.f); + BOOST_CHECK_EQUAL(vw2_updated->sd->weighted_labeled_examples, 2.f); +} diff --git a/unit_test/watchdog_test.cc b/unit_test/watchdog_test.cc index 716a118d1..1a6368a3e 100644 --- a/unit_test/watchdog_test.cc +++ b/unit_test/watchdog_test.cc @@ -1,11 +1,12 @@ #ifdef STAND_ALONE # define BOOST_TEST_MODULE Main #endif -#include "utility/watchdog.h" + #include #include "common_test_utils.h" #include "str_util.h" +#include "utility/watchdog.h" #include #include @@ -49,7 +50,7 @@ BOOST_AUTO_TEST_CASE(watchdog_unregister) BOOST_AUTO_TEST_CASE(watchdog_fail_after_several_iterations) { - if (is_invoked_with("valgrind")) + if (test_utils::is_invoked_with("valgrind")) { // this test depends on clock timeouts, can't guarantee test success under valgrind std::cout << "skipping watchdog_fail_after_several_iterations test when running in valgrind" << std::endl; @@ -98,7 +99,7 @@ BOOST_AUTO_TEST_CASE(watchdog_report_with_error_handler) BOOST_AUTO_TEST_CASE(watchdog_multiple_threads) { - if (is_invoked_with("valgrind")) + if (test_utils::is_invoked_with("valgrind")) { // this test depends on clock timeouts, can't guarantee test success under valgrind std::cout << "skipping watchdog_multiple_threads test when running in valgrind" << std::endl; From 880de9e221ebdfcc96f465bbc15e7d50ba65246a Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Thu, 2 Nov 2023 14:06:20 -0400 Subject: [PATCH 12/15] rl_sim update --- examples/rl_sim_cpp/local_loop_client.json | 16 +++++ examples/rl_sim_cpp/main.cc | 34 +++++++---- examples/rl_sim_cpp/rl_sim.cc | 69 +++++++++++++++++++--- examples/rl_sim_cpp/rl_sim.h | 3 +- examples/rl_sim_cpp/simulation_stats.h | 4 ++ 5 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 examples/rl_sim_cpp/local_loop_client.json diff --git a/examples/rl_sim_cpp/local_loop_client.json b/examples/rl_sim_cpp/local_loop_client.json new file mode 100644 index 000000000..dd89a3829 --- /dev/null +++ b/examples/rl_sim_cpp/local_loop_client.json @@ -0,0 +1,16 @@ +{ + "model.vw.initial_command_line": "--cb_explore_adf --json --epsilon 0.0 --preserve_performance_counters -q :: --driver_output_off", + "ApplicationID": "local_loop", + "IsExplorationEnabled": true, + "InitialExplorationEpsilon": 0.2, + "protocol.version": "2", + "model.source": "LOCAL_LOOP_MODEL_DATA", + "interaction.sender.implementation": "LOCAL_LOOP_SENDER", + "observation.sender.implementation": "LOCAL_LOOP_SENDER", + "eud.duration": "0:0:1", + "joiner.problem.type": "PROBLEM_TYPE_CB", + "joiner.reward.function": "REWARD_FUNCTION_EARLIEST", + "joiner.learning.mode": "ONLINE", + "model.refreshintervalms": "5000", + "time_provider.implementation": "CLOCK_TIME_PROVIDER" + } \ No newline at end of file diff --git a/examples/rl_sim_cpp/main.cc b/examples/rl_sim_cpp/main.cc index 5852fbaef..406537c79 100644 --- a/examples/rl_sim_cpp/main.cc +++ b/examples/rl_sim_cpp/main.cc @@ -26,19 +26,27 @@ int main(int argc, char** argv) po::variables_map process_cmd_line(const int argc, char** argv) { po::options_description desc("Options"); - desc.add_options()("help", "produce help message")("json_config,j", - po::value()->default_value("client.json"), "JSON file with config information for hosted RL loop")( - "log_to_file,l", po::value()->default_value(false), "Log interactions and observations to local files")( - "get_model,m", po::value()->default_value(true), "Download model from model source")( - "log_timestamp,t", po::value()->default_value(true), "Apply timestamp to all logged message")("ccb", - po::value()->default_value(false), "Run in ccb mode")("slates", po::value()->default_value(false), - "Run in slates mode")("ca", po::value()->default_value(false), "Run in continuous actions mode")( - "multistep", po::value()->default_value(false), "Run in multistep mode")( - "num_events", po::value()->default_value(0), "Number of event series' to be sent. 0 is infinite.")( - "random_seed", po::value()->default_value(rand()), "Random seed. Default is random")( - "delay", po::value()->default_value(2000), "Delay between events in ms")( - "quiet", po::bool_switch(), "Suppress logs")("random_ids", po::value()->default_value(true), - "Use randomly generated Event IDs. Default is true")("throughput", "print throughput stats"); + desc.add_options()( + "help", "produce help message")( + "json_config,j", po::value()->default_value("client.json"), + "JSON file with config information for hosted RL loop")( + "log_to_file,l", po::value()->default_value(false), "Log interactions and observations to local files")( + "get_model,m", po::value()->default_value(true), "Download model from model source")( + "log_timestamp,t", po::value()->default_value(true), "Apply timestamp to all logged message")( + "ccb", po::value()->default_value(false), "Run in ccb mode")( + "slates", po::value()->default_value(false), "Run in slates mode")( + "ca", po::value()->default_value(false), "Run in continuous actions mode")( + "multistep", po::value()->default_value(false), "Run in multistep mode")( + "num_events", po::value()->default_value(0), "Number of event series' to be sent. 0 is infinite.")( + "random_seed", po::value()->default_value(rand()), "Random seed. Default is random")( + "delay", po::value()->default_value(2000), "Delay between events in ms")( + "quiet", po::bool_switch(), "Suppress logs")( + "random_ids", po::value()->default_value(true), "Use randomly generated Event IDs. Default is true")( + "throughput", "print throughput stats")( + "random_ids", po::value()->default_value(true), "Use randomly generated Event IDs. Default is true")( + "refresh_model_period", po::value()->default_value(0), + "Call refresh model after every N examples. 0 turns off explicit model refresh and relies on background refresh. " + "Must disable background refresh in client.json with key 'model.backgroundrefresh'"); po::variables_map vm; store(parse_command_line(argc, argv, desc), vm); diff --git a/examples/rl_sim_cpp/rl_sim.cc b/examples/rl_sim_cpp/rl_sim.cc index 724de55a5..467f69d8a 100644 --- a/examples/rl_sim_cpp/rl_sim.cc +++ b/examples/rl_sim_cpp/rl_sim.cc @@ -88,8 +88,19 @@ int rl_sim::cb_loop() { std::cout << " " << stats.count() << ", ctxt, " << p.id() << ", action, " << chosen_action << ", outcome, " << outcome << ", dist, " << get_dist_str(response) << ", " << stats.get_stats(p.id(), chosen_action) - << std::endl; + << ", ctr: " << stats.get_ctr() << std::endl; } + // refresh model every _model_refresh_period events + std::cerr << "Current events: " << _current_events << std::endl; + if (_model_refresh_period != 0 && (_current_events % _model_refresh_period) == 0) + { + r::api_status status; + if (_rl->refresh_model(&status) != err::success) + { + std::cout << status.get_error_msg() << std::endl; + continue; + } + } std::this_thread::sleep_for(std::chrono::milliseconds(_delay)); } @@ -165,6 +176,18 @@ int rl_sim::multistep_loop() { std::cout << status.get_error_msg() << std::endl; continue; + + } + // refresh model every _model_refresh_period events + // Treat each episode as a single event + if (_model_refresh_period != 0 && (_current_events / episode_length) % _model_refresh_period == 0) + { + r::api_status status; + if (_rl->refresh_model(&status) != err::success) + { + std::cout << status.get_error_msg() << std::endl; + continue; + } } std::this_thread::sleep_for(std::chrono::milliseconds(_delay)); @@ -207,6 +230,17 @@ int rl_sim::ca_loop() << stats.get_stats(joint.id(), chosen_action) << std::endl; } + // refresh model every _model_refresh_period events + if (_model_refresh_period != 0 && _current_events % _model_refresh_period == 0) + { + r::api_status status; + if (_rl->refresh_model(&status) != err::success) + { + std::cout << status.get_error_msg() << std::endl; + continue; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(_delay)); } return 0; @@ -260,6 +294,17 @@ int rl_sim::ccb_loop() } index++; } + + // refresh model every _model_refresh_period events + if (_model_refresh_period != 0 && _current_events % _model_refresh_period == 0) + { + r::api_status status; + if (_rl->refresh_model(&status) != err::success) + { + std::cout << status.get_error_msg() << std::endl; + continue; + } + } std::this_thread::sleep_for(std::chrono::milliseconds(_delay)); } @@ -331,6 +376,17 @@ int rl_sim::slates_loop() continue; } + // refresh model every _model_refresh_period events + if (_model_refresh_period != 0 && _current_events % _model_refresh_period == 0) + { + r::api_status status; + if (_rl->refresh_model(&status) != err::success) + { + std::cout << status.get_error_msg() << std::endl; + continue; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(_delay)); } @@ -454,11 +510,7 @@ int rl_sim::init_rl() config.set(r::name::OBSERVATION_SENDER_IMPLEMENTATION, r::value::OBSERVATION_FILE_SENDER); } - if (!_options["get_model"].as()) - { - // Set the time provider to the clock time provider - config.set(r::name::MODEL_SRC, r::value::NO_MODEL_DATA); - } + if (!_options["get_model"].as()) { config.set(r::name::MODEL_SRC, r::value::NO_MODEL_DATA); } if (_options["log_timestamp"].as()) { @@ -628,7 +680,7 @@ std::string rl_sim::create_context_json(const std::string& cntxt, const std::str std::string rl_sim::create_event_id() { - if (_num_events > 0 && ++_current_events >= _num_events) { _run_loop = false; } + if (++_current_events >= _num_events && _num_events > 0) { _run_loop = false; } if (_random_ids) { return boost::uuids::to_string(boost::uuids::random_generator()()); } @@ -637,7 +689,7 @@ std::string rl_sim::create_event_id() return oss.str(); } -rl_sim::rl_sim(boost::program_options::variables_map vm) : _options(std::move(vm)), _loop_kind(CB) +rl_sim::rl_sim(const boost::program_options::variables_map& vm) : _options(vm), _loop_kind(CB) { if (_options["ccb"].as()) { _loop_kind = CCB; } else if (_options["slates"].as()) { _loop_kind = Slates; } @@ -649,6 +701,7 @@ rl_sim::rl_sim(boost::program_options::variables_map vm) : _options(std::move(vm _delay = _options["delay"].as(); _quiet = _options["quiet"].as(); _random_ids = _options["random_ids"].as(); + _model_refresh_period = _options["refresh_model_period"].as(); } std::string get_dist_str(const reinforcement_learning::ranking_response& response) diff --git a/examples/rl_sim_cpp/rl_sim.h b/examples/rl_sim_cpp/rl_sim.h index 8e583e62d..3a0db2c25 100644 --- a/examples/rl_sim_cpp/rl_sim.h +++ b/examples/rl_sim_cpp/rl_sim.h @@ -26,7 +26,7 @@ class rl_sim * * @param vm User defined options */ - explicit rl_sim(boost::program_options::variables_map vm); + explicit rl_sim(const boost::program_options::variables_map& vm); /** * @brief Simulation loop @@ -177,4 +177,5 @@ class rl_sim int64_t _delay = 2000; bool _quiet = false; bool _random_ids = true; + uint64_t _model_refresh_period = 0; }; diff --git a/examples/rl_sim_cpp/simulation_stats.h b/examples/rl_sim_cpp/simulation_stats.h index 237710d3f..26896fcc4 100644 --- a/examples/rl_sim_cpp/simulation_stats.h +++ b/examples/rl_sim_cpp/simulation_stats.h @@ -17,6 +17,7 @@ class simulation_stats auto& item_count = _item_stats[id]; ++item_count; ++_total_events; + _total_reward += outcome; } std::string get_stats(const std::string& id, T chosen_action) @@ -29,8 +30,11 @@ class simulation_stats int count() const { return _total_events; } + float get_ctr() const { return _total_reward / _total_events; } + private: std::map, std::pair> _action_stats; std::map _item_stats; int _total_events = 0; + float _total_reward = 0.f; }; \ No newline at end of file From dc06e4547329173e6f0a0cd7f5529c14d51429f3 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Thu, 2 Nov 2023 14:11:51 -0400 Subject: [PATCH 13/15] readme update --- external_parser/README.md | 180 +++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 63 deletions(-) diff --git a/external_parser/README.md b/external_parser/README.md index 93e680741..834dd0b95 100644 --- a/external_parser/README.md +++ b/external_parser/README.md @@ -1,98 +1,157 @@ # Parser and reward calculator for joined binary schema v2 files -## File format +## Binary log file format +The binary log file consists of a sequence of *messages*, with each message being one of five types. The message types are FILEMAGIC, HEADER, REGULAR, CHECKPOINT, and EOF. The binary log should begin with a FILEMAGIC message and end with an EOF message, although this is not mandatory. -The file format constitute of a sequence of messages writen one after the other. All messages have the same format: +Each message contains a *payload*, except for the FILEMAGIC message which has an *inline payload*. For an inline payload, the payload size field is used to store the payload data itself (which must be 4 bytes long), and the later fields are omitted. -- 4 bytes - message type -- 4 bytes - payload size or inline payload. -- bytes - payload content _(optional)_ -- padding bytes - size % 8 bytes to ensure message alignment _(optional)_ +The general format of each message is shown in this table. Details for each message type are described later. -The size of all message must be aligned to 8 bytes. Paddings bytes are inserted at the end and should not be included in the payload size. -Padding bytes should be zero but is not enforced. +| Size (bytes) | Description | +| ---- | ----------- | +| 4 | message type identifier | +| 4 | payload size (or inline payload data) | +| `payload_size` | payload content (required if not inline payload) | +| `payload_size % 8` | padding (required if not inline payload) | -### Message types - -Each message has an unique indentifier, those are the ones currently recognized: - -- `MSG_TYPE_FILEMAGIC = 0x42465756 //'VWFB'` +### Message type +The first 4 bytes in each message is an unique indentifier: +- `MSG_TYPE_FILEMAGIC = 0x42465756` (in ASCII this is `VWFB`) - `MSG_TYPE_HEADER = 0x55555555` - `MSG_TYPE_REGULAR = 0xFFFFFFFF` - `MSG_TYPE_CHECKPOINT = 0x11111111` - `MSG_TYPE_EOF = 0xAAAAAAAA` -### Message payloads +### Payload size +For all message types except the FILEMAGIC message, the payload size field should store the size of the payload as an unsigned 32-bit integer. We will refer to this value as `payload_size`. -For all flatbuffer payloads see `rlclientlib/schema/v2` +For the FILEMAGIC message, the payload size field is used to store an inline payload containing the version of the binary log file format. As of now the only existing version is `1` (one), so the FILEMAGIC message must have its payload size field equal to one. -### File Magic message +Note that the binary log format version is not the same as the Flatbuffer schema version, which is version 2 for all the flatbuffers described here. + +### Payload content +The following `payload_size` bytes of the file are interpreted as the payload data itself. -The payload is inline and it's the file format version. +### Padding +The file format requires `payload_size % 8` bytes of padding at the end of every payload. Note that this does not align the total message size to a multiple of 8 bytes. (For example a one byte payload gets one padding byte appended to make 2 bytes total.) It is merely a file format requirement and is necessary for the next message to be correctly parsed. + +Padding bytes should not be included when computing `payload_size`. Padding bytes should have a value of zero, but this is not enforced. + +### Recomended message ordering +The recomended ordering of messages in a file is shown here. -The only value accepted is `1`. +- 1 FILEMAGIC message +- 1 HEADER message +- `N` times: + - 1 CHECKPOINT message + - `M` REGULAR messages (to be processed with the same checkpoint information) +- 1 EOF message -This message should be the first on a file, making it easy to recognize files following its format by their 4 bytes watermark. +Only the REGULAR message is strictly necessary. Although it is not recommended, a binary log without the other message types can still be parsed correctly. + +## Message details +The following gives details about each message type. For information on flatbuffer payload schemas, see [`rlclientlib/schema/v2`](https://github.com/VowpalWabbit/reinforcement_learning/tree/master/rlclientlib/schema/v2) + +### File Magic message +The FILEMAGIC message does not contain a normal payload but instead must have `payload_size = 1` as described above. + +The FILEMAGIC message is optional, but it is recommended to begin every binary log file with it. ### Header message +The payload for a HEADER message is a flatbuffer of type `FileHeader`. The HEADER message is optional and does not affect how the binary log is parsed. -Payload is a flatbuffer message of type `FileHeader` (see `FileFormat.fbs`) . +``` +table FileHeader { + join_time: TimeStamp; + properties: [KeyValue]; +} +``` +(see [`FileFormat.fbs`](https://github.com/VowpalWabbit/reinforcement_learning/blob/master/rlclientlib/schema/v2/FileFormat.fbs)) -This message contains informational data about this file. It should include provenance -details such as how it was generated, the version and parameters of the program used, generation -time and other information that helps troubleshooting. +This message contains informational data about this file. It should include provenance details such as how it was generated, the version and parameters of the program used, generation time, and other information that helps troubleshooting. ### Checkpoint message +The payload for a CHECKPOINT message is a flatbuffer of type `CheckpointInfo`. + +``` +table CheckpointInfo { + reward_function_type: RewardFunctionType; + default_reward: float; + learning_mode_config: LearningModeType; + problem_type_config: ProblemType; + use_client_time: bool; +} +``` +(see [`FileFormat.fbs`](https://github.com/VowpalWabbit/reinforcement_learning/blob/master/rlclientlib/schema/v2/FileFormat.fbs)) -Payload is a flatbuffer message of type `CheckpointInfo` (see `FileFormat.fbs`). +This message includes information on how to join events comming after it. If a large binary log file must be split, it is recommended to split at the start of a CHECKPOINT message as they contain all info required to process the stream that follows it. -This message includes information on how to join events comming after it. -Checkpoint messages are the recomended point to split larger files at as they contain all info -required to process the stream that follows them. +The CHECKPOINT message is optional. If it is not present in a binary log, events will be processed with the default joiner configuration. If it is only present after encountering any REGULAR messages, all the preceding REGULAR messages will be processed with the default joiner configuration. ### Regular message +The payload for a REGULAR message is a flatbuffer of type `JoinedPayload`. -Payload is a flatbuffer message of type `JoinedPayload` (see `FileFormat.fbs`). +``` +table JoinedPayload { + events: [JoinedEvent]; +} -This message include multiple events, sharing one or more event-ids that should be processed together. +table JoinedEvent { + event: [ubyte]; + timestamp: TimeStamp; +} +``` +(see [`FileFormat.fbs`](https://github.com/VowpalWabbit/reinforcement_learning/blob/master/rlclientlib/schema/v2/FileFormat.fbs)) -### Recomended message ordering +The `JoinedPayload` flatbuffer internally contains a vector of `JoinedEvent`s. All of these `JoinedEvent`s should share the same event ID. For a binary log containing more than one event ID, multiple REGULAR messages should be used with one for each ID. -The recomended ordering of messages in a file is the following: +The `JoinedEvent` flatbuffer has an `event` field that contains a serialized `Event` flatbuffer. The `Event` flatbuffer contains a metadata field and a `payload` field that contains a serialized type-specific event flatbuffer (CaEvent, CbEvent, OutcomeEvent, etc.). In other words, the final event data is contained in a flatbuffer (type specific) inside a flatbuffer (Event) inside a flatbuffer (JoinedPayload) inside a custom message format (REGULAR message). -- 1 file magic message -- 1 file header message -N times: -- 1 checkpoint message -- M regular messages +``` +table Event { + meta:Metadata; + payload:[ubyte]; +} +table Metadata { + id:string; + client_time_utc:TimeStamp; + app_id:string; + payload_type:PayloadType; + pass_probability:float; + encoding: EventEncoding; +} +``` +(see [`Event.fbs`](https://github.com/VowpalWabbit/reinforcement_learning/blob/master/rlclientlib/schema/v2/Event.fbs) and [`Metadata.fbs`](https://github.com/VowpalWabbit/reinforcement_learning/blob/master/rlclientlib/schema/v2/Metadata.fbs)) -## Linux +### EOF message +The EOF (End Of File) message stops file parsing as soon as it is read. It should contain a payload size of zero and no payload, but any non-zero payload will be simply ignored as parsing will stop when the EOF message identifier is read. -### Build Linux +The EOF message is optional, but it is recommended to end every binary log file with it. If it is not present, parsing will continue until the end of the input data is reached or an error is encountered. -**Note**: To statically link set `-DSTATIC_LINK_BINARY_PARSER=ON` during `cmake` +# Build instructions +The binary parser may be build either along with rlclientlib or as a standalone project. -from `external_parser`: +## Building with rlclientlib +To build with rlclientlib, add `-DRL_BUILD_EXTERNAL_PARSER=On` to the CMake command line when configuring rlclientlib. Compiling rlclientlib will then produce an executable at `build/external_parser/vw`. -- mkdir build -- cd build -- cmake .. -- make -j $(nproc) +The following instructions assume building the parser as a standalone project. -vw executable located at: `external_parser/build/vw` +## Linux +### Build +**Note**: To statically link, add `-DSTATIC_LINK_BINARY_PARSER=ON` to the `cmake` command line -### Run +Run these commands from `external_parser` directory: +- `cmake -S . -B build` +- `cmake --build build` -`./vw -d --binary_parser [other vw args]` +The output is a `vw` executable located at: `external_parser/build/vw` +### Run +`./vw -d --binary_parser [other vw args]` ## Windows - -cmake build for windows - -### Deps: - +### Install dependencies **Note**: to link statically then replace `x64-windows` with `x64-windows-static-md` during vcpkg installation and in the cmake `-DVCPKG_TARGET_TRIPLET` **Note**: vcpkg doesn't play well with nugets in visual studio so if you are trying to build something else in visual studio that uses the below via nugets you might get linking errors @@ -103,19 +162,14 @@ cmake build for windows - `vcpkg install boost-test:x64-windows` - `vcpkg install flatbuffers:x64-windows` -### Build: - -from `external_parser` (replace `Release` with `Debug` if you want a debug build): - -- mkdir build -- cd build -- cmake .. -DCMAKE_TOOLCHAIN_FILE=\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows -G "Visual Studio 15 2017" -A x64 -DBUILD_FLATBUFFERS=OFF -DCMAKE_CONFIGURATION_TYPES="Release" -DWARNINGS=OFF -DWARNINGS=OFF -DWARNING_AS_ERROR=OFF -DDO_NOT_BUILD_VW_C_WRAPPER=OFF -DBUILD_JAVA=OFF -DBUILD_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXPERIMENTAL_BINDING=OFF -- /verbosity:normal /m /p:Configuration=Release;Platform=x64 vw_binary_parser.sln +### Build +Run these commands from `external_parser` (replace `Release` directory with `Debug` if you want a debug build): +- `cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows -G "Visual Studio 15 2017" -A x64 -DBUILD_FLATBUFFERS=OFF -DCMAKE_CONFIGURATION_TYPES="Release" -DWARNINGS=OFF -DWARNINGS=OFF -DWARNING_AS_ERROR=OFF -DDO_NOT_BUILD_VW_C_WRAPPER=OFF -DBUILD_JAVA=OFF -DBUILD_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXPERIMENTAL_BINDING=OFF` +- ` /verbosity:normal /m /p:Configuration=Release;Platform=x64 vw_binary_parser.sln` `vw_binary_parser.sln` will be available under `build` and can be used to open the solution in visual studio -vw executable located at: `external_parser\build\Release\vw.exe` +The output is a `vw` executable located at: `external_parser\build\Release\vw.exe` ### Run - -`.\Release\vw.exe -d --binary_parser [other vw args]` +`.\build\Release\vw.exe -d --binary_parser [other vw args]` From cc388e71ab75a5366357087e04c6ec0bf7d9dab9 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Thu, 2 Nov 2023 14:12:53 -0400 Subject: [PATCH 14/15] linux ci update --- .scripts/linux/run-clang-tidy.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.scripts/linux/run-clang-tidy.sh b/.scripts/linux/run-clang-tidy.sh index 6ddf3fe90..8c6df475b 100755 --- a/.scripts/linux/run-clang-tidy.sh +++ b/.scripts/linux/run-clang-tidy.sh @@ -17,8 +17,7 @@ cd "$REPO_DIR" cmake -S . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=On # generate flatbuffers files -cmake --build build --target fbgenerator_v1 -cmake --build build --target fbgenerator_v2 +cmake --build build --target fbgen # check that compile_commands.json was generated cd build From ee64dc9cb199e3f589bf4d3e34105be5c8394f45 Mon Sep 17 00:00:00 2001 From: "ataymano@microsoft.com" Date: Thu, 2 Nov 2023 15:33:06 -0400 Subject: [PATCH 15/15] rl_sim fix --- examples/rl_sim_cpp/main.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/rl_sim_cpp/main.cc b/examples/rl_sim_cpp/main.cc index 406537c79..07c678604 100644 --- a/examples/rl_sim_cpp/main.cc +++ b/examples/rl_sim_cpp/main.cc @@ -41,7 +41,6 @@ po::variables_map process_cmd_line(const int argc, char** argv) "random_seed", po::value()->default_value(rand()), "Random seed. Default is random")( "delay", po::value()->default_value(2000), "Delay between events in ms")( "quiet", po::bool_switch(), "Suppress logs")( - "random_ids", po::value()->default_value(true), "Use randomly generated Event IDs. Default is true")( "throughput", "print throughput stats")( "random_ids", po::value()->default_value(true), "Use randomly generated Event IDs. Default is true")( "refresh_model_period", po::value()->default_value(0),