v4.17.0 (#15)

CMakeLists.txt

@@ -1,8 +1,12 @@
 cmake_minimum_required(VERSION 3.0.0)
 
 find_program(CCACHE_PROGRAM ccache)
+find_program(CLACHE_PROGRAM clcache)
+
 if(CCACHE_PROGRAM)
     set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
+elseif(CLCACHE_PROGRAM)
+    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CLCACHE_PROGRAM}")
 endif()
 
 project(HailoRT)

common/include/context_switch_defs.h

@@ -56,6 +56,13 @@ extern "C" {
 
 #define CONTEXT_SWITCH_DEFS__WRITE_ACTION_BY_TYPE_MAX_SIZE (4)
 
+// TODO HRT-12512: Update variable when / If DDR has it's own CMA region
+#define CONTEXT_SWITCH_DEFS__START_M4_MAPPED_DDR_ADDRESS (0x80000000)
+#define CONTEXT_SWITCH_DEFS__END_M4_MAPPED_DDR_ADDRESS (0x90000000)
+#define CONTEXT_SWITCH_DEFS__START_M4_MAPPED_DDR_ADDRESS_AFTER_LUT (0x50000000)
+#define CONTEXT_SWITCH_DEFS__DDR_ADDRESS_MASK (0x0FFFFFFF)
+#define CONTEXT_SWITCH_DEFS__INVALID_DDR_CONTEXTS_BUFFER_ADDRESS (0)
+
 
 #pragma pack(push, 1)
 typedef struct {
@@ -207,7 +214,7 @@ typedef struct {
     uint32_t kernel_done_count;
 } CONTEXT_SWITCH_DEFS__enable_lcu_action_non_default_data_t;
 
-/* Default action - kernel_done_address and kernel_done_count has default values */
+/* Default action - kernel_done_address, kernel_done_count have default values */
 typedef struct {
     uint8_t packed_lcu_id;
     uint8_t network_index;

common/include/control_protocol.h

@@ -81,6 +81,7 @@ extern "C" {
 /* Value to represent an operation should be performed on all streams. */
 #define CONTROL_PROTOCOL__ALL_DATAFLOW_MANAGERS (0xFF)
 
+#define CONTROL_PROTOCOL__MAX_CONTEXT_SIZE (3072)
 
 #define CONTROL_PROTOCOL__OPCODES_VARIABLES \
     CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_IDENTIFY,                                  true,  CPU_ID_APP_CPU)\
@@ -868,15 +869,18 @@ typedef struct {
 
 typedef struct {
     bool preliminary_run_asap;
+    bool batch_register_config;
+    bool can_fast_batch_switch;
 } CONTROL_PROTOCOL__INFER_FEATURE_LIST_t;
 
 typedef struct {
-    uint8_t dynamic_contexts_count;
+    uint16_t dynamic_contexts_count;
     CONTROL_PROTOCOL__INFER_FEATURE_LIST_t infer_features;
     CONTROL_PROTOCOL__VALIDATION_FEATURE_LIST_t validation_features;
     uint8_t networks_count;
     uint16_t csm_buffer_size;
     uint16_t batch_size[CONTROL_PROTOCOL__MAX_NETWORKS_PER_NETWORK_GROUP];
+    uint32_t external_action_list_address;
     uint32_t boundary_channels_bitmap[CONTROL_PROTOCOL__MAX_VDMA_ENGINES_COUNT];
 } CONTROL_PROTOCOL__application_header_t;
 
@@ -954,10 +958,10 @@ typedef struct {
 #pragma warning(disable: 4200)
 #endif
 typedef struct {
-    uint32_t is_first_control_per_context_length;
+    uint32_t is_first_chunk_per_context_length;
-    uint8_t is_first_control_per_context;
+    uint8_t is_first_chunk_per_context;
-    uint32_t is_last_control_per_context_length;
+    uint32_t is_last_chunk_per_context_length;
-    uint8_t is_last_control_per_context;
+    uint8_t is_last_chunk_per_context;
     uint32_t context_type_length;
     uint8_t context_type; // CONTROL_PROTOCOL__context_switch_context_type_t
     uint32_t context_network_data_length;
@@ -988,7 +992,7 @@ typedef struct {
     uint32_t context_type_length;
     uint8_t context_type; // CONTROL_PROTOCOL__context_switch_context_type_t
     uint32_t context_index_length;
-    uint8_t context_index;
+    uint16_t context_index;
     uint32_t action_list_offset_length;
     uint16_t action_list_offset;
 } CONTROL_PROTOCOL__download_context_action_list_request_t;
@@ -1160,7 +1164,7 @@ typedef struct {
     bool break_at_any_batch_index;
     uint16_t batch_index;
     bool break_at_any_context_index;
-    uint8_t context_index;
+    uint16_t context_index;
     bool break_at_any_action_index;
     uint16_t action_index;
 } CONTROL_PROTOCOL__context_switch_breakpoint_data_t;
@@ -1470,15 +1474,21 @@ typedef enum {
     CONTROL_PROTOCOL__CONTEXT_SWITCH_INDEX_COUNT,
 } CONTROL_PROTOCOL__context_switch_context_index_t;
 
-#define CONTROL_PROTOCOL__MAX_CONTEXTS_PER_NETWORK_GROUP (64)
+#define CONTROL_PROTOCOL__MAX_CONTEXTS_PER_NETWORK_GROUP (1024)
 
+// This struct will be used for both ControlActionList and DDRActionlist (in order to keep flow in FW as similar as possible)
+// The context_network_data array will never have more data than CONTROL_PROTOCOL__CONTEXT_NETWORK_DATA_SINGLE_CONTROL_MAX_SIZE
+// In case of ControlActionList - this is verified when sending and receiving control. We make it larger here to be
+// able to hold DDRActionList Contexts without needing to copy or do more processing in fw.
+// In both cases this struct holds a chunk of the context - in ControlActionList - it will be as much of the context a
+// Single control message is able to carry and in DDRActionlist will be the whole context
 typedef struct {
-    bool is_first_control_per_context;
+    bool is_first_chunk_per_context;
-    bool is_last_control_per_context;
+    bool is_last_chunk_per_context;
     uint8_t context_type; // CONTROL_PROTOCOL__context_switch_context_type_t
     uint32_t context_network_data_length;
-    uint8_t context_network_data[CONTROL_PROTOCOL__CONTEXT_NETWORK_DATA_SINGLE_CONTROL_MAX_SIZE];
+    uint8_t context_network_data[CONTROL_PROTOCOL__MAX_CONTEXT_SIZE];
-} CONTROL_PROTOCOL__context_switch_context_info_single_control_t;
+} CONTROL_PROTOCOL__context_switch_context_info_chunk_t;
 
 CASSERT(sizeof(CONTROL_PROTOCOL__context_switch_context_index_t)<=UINT8_MAX, control_protocol_h);
 CASSERT(sizeof(CONTROL_PROTOCOL__context_switch_context_type_t)<=UINT8_MAX, control_protocol_h);

common/include/d2h_events.h

@@ -123,14 +123,14 @@ typedef struct {
 
 #define D2H_EVENT_HEALTH_MONITOR_CPU_ECC_EVENT_PARAMETER_COUNT  (1)
 
-/* D2H_EVENT_context_switch_breakpoint_reached_event_massage_t should be the same as 
+/* D2H_EVENT_context_switch_breakpoint_reached_event_message_t should be the same as 
  * CONTROL_PROTOCOL__context_switch_breakpoint_data_t and hailo_context_switch_breakpoint_reached_notification_message_t */
 typedef struct {
     uint8_t application_index;
     uint16_t batch_index;
-    uint8_t context_index;
+    uint16_t context_index;
     uint16_t action_index;
-} D2H_EVENT_context_switch_breakpoint_reached_event_massage_t;
+} D2H_EVENT_context_switch_breakpoint_reached_event_message_t;
 
 #define D2H_EVENT_CONTEXT_SWITCH_BREAKPOINT_REACHED_EVENT_PARAMETER_COUNT  (4)
 
@@ -151,7 +151,7 @@ typedef struct {
     uint32_t exit_status;
     uint8_t application_index;
     uint16_t batch_index;
-    uint8_t context_index;
+    uint16_t context_index;
     uint16_t action_index;
 } D2H_EVENT_context_switch_run_time_error_event_message_t;
 
@@ -166,7 +166,7 @@ typedef union {
    D2H_EVENT_health_monitor_overcurrent_alert_event_message_t health_monitor_overcurrent_alert_event;
    D2H_EVENT_health_monitor_lcu_ecc_error_event_message_t health_monitor_lcu_ecc_error_event;
    D2H_EVENT_health_monitor_cpu_ecc_event_message_t health_monitor_cpu_ecc_event;
-   D2H_EVENT_context_switch_breakpoint_reached_event_massage_t context_switch_breakpoint_reached_event;
+   D2H_EVENT_context_switch_breakpoint_reached_event_message_t context_switch_breakpoint_reached_event;
    D2H_EVENT_health_monitor_clock_changed_event_message_t health_monitor_clock_changed_event;
    D2H_EVENT_hw_infer_mamager_infer_done_message_t hw_infer_manager_infer_done_event;
    D2H_EVENT_context_switch_run_time_error_event_message_t context_switch_run_time_error_event;

common/include/firmware_status.h

@@ -527,6 +527,7 @@ Updating rules:
    FIRMWARE_STATUS__X(QSPI_STATUS_MISALIGNED_ADDRESS)\
    FIRMWARE_STATUS__X(QSPI_STATUS_BLOCK_ERASE_FAILED)\
    FIRMWARE_STATUS__X(QSPI_STATUS_CLEAR_AHB_REMAP_FAILED)\
+   FIRMWARE_STATUS__X(QSPI_STATUS_NOT_SUPPORTED)\
    \
    FIRMWARE_MODULE__X(FIRMWARE_MODULE__PCIE_SERVICE)\
    FIRMWARE_STATUS__X(PCIE_SERVICE_STATUS_INVALID_PARAMETERS)\
@@ -763,6 +764,7 @@ Updating rules:
    FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_WRITE_DATA_BY_TYPE_ACTION_INVALID_TYPE)\
    FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_WRITE_DATA_BY_TYPE_ACTION_INVALID_MEMORY_SPACE)\
    FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_REACHED_TIMEOUT_WHILE_WAITING_FOR_BATCH_SWITCH_CONTEXT_TO_END)\
+   FIRMWARE_STATUS__X(CONTEXT_SWITCH_STATUS_INVALID_EXTERNAL_ACTION_LIST_ADDRESS)\
    \
    FIRMWARE_MODULE__X(FIRMWARE_MODULE__D2H_EVENT_MANAGER)\
    FIRMWARE_STATUS__X(HAILO_D2H_EVENT_MANAGER_STATUS_MESSAGE_HIGH_PRIORITY_QUEUE_CREATE_FAILED)\
@@ -1080,6 +1082,7 @@ Updating rules:
    FIRMWARE_STATUS__X(BURST_CREDITS_TASK_STATUS_FAILED_TO_FIND_STREAM_INDEX)\
    FIRMWARE_STATUS__X(BURST_CREDITS_TASK_STATUS_TASK_NO_CONFIGURED_ACTIONS)\
    FIRMWARE_STATUS__X(BURST_CREDITS_TASK_STATUS_TASK_EXPECTED_HIGHER_BATCH)\
+   FIRMWARE_STATUS__X(BURST_CREDITS_TASK_STATUS_TASK_REACHED_TIMEOUT_WAITING_FOR_DEACTIVATION)\
    \
    FIRMWARE_MODULE__X(FIRMWARE_MODULE__TASK_SYNC_EVENTS)\
    FIRMWARE_STATUS__X(TASK_SYNC_EVENTS_STATUS_START_TASK_WHILE_IT_IS_RUNNING)\
@@ -1111,13 +1114,19 @@ Updating rules:
    FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_RECEIVED_UNEXPECTED_INTERRUPT)\
    FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_NETWORK_INDEX)\
    FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_KERNEL_DONE_COUNT)\
+   FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_EXTENSION)\
    \
    FIRMWARE_MODULE__X(FIRMWARE_MODULE__HW_INFER_MANAGER)\
    FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_NETWORK_GROUP_NOT_CONFIGURED_BEFORE_INFER_START)\
    FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_NETWORK_GROUP_ALREADY_ACTIVATED)\
    FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_STATE_MACHINE_NOT_IN_RESET_STATE_BEFORE_DEACTIVATE)\
    FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_INVALID_STATE)\
-
+   \
+   FIRMWARE_MODULE__X(FIRMWARE_MODULE__INFINITE_CONTEXT_LOADER)\
+   FIRMWARE_STATUS__X(INFINITE_CONTEXT_LOADER_STATUS_EVENT_BITS_NOT_CLEARED_BEFORE_COPY_CALL)\
+   FIRMWARE_STATUS__X(INFINITE_CONTEXT_LOADER_STATUS_TIMEOUT_OCCURED_WAITING_FOR_COPY)\
+   FIRMWARE_STATUS__X(INFINITE_CONTEXT_LOADER_STATUS_NOT_SUPPORTED)\
+   FIRMWARE_STATUS__X(INFINITE_CONTEXT_LOADER_STATUS_NOT_MODULE_NOT_INITIALIZED)\
 
 typedef enum {
 #define FIRMWARE_MODULE__X(module) module,

common/include/utils.h

@@ -54,6 +54,10 @@
 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
 #endif
 
+#ifndef DIV_ROUND_DOWN
+#define DIV_ROUND_DOWN(n,d) ((n) / (d))
+#endif
+
 #ifndef ROUND_UNSIGNED_FLOAT
 #define ROUND_UNSIGNED_FLOAT(n) ((n - (uint32_t)(n)) > 0.5) ? (uint32_t)(n + 1) : (uint32_t)(n)
 #endif

hailort/.gitignore

@@ -1,3 +1,4 @@
+build/
+dist/
 /external/
 cmake/external/*/
-prepare_externals/build/

hailort/CMakeLists.txt

@@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 option(HAILO_BUILD_PYBIND "Build Python binding" OFF)
 option(HAILO_BUILD_EMULATOR "Build hailort for emulator" OFF)
 option(HAILO_BUILD_UT "Build Unit Tests" OFF)
+option(HAILO_BUILD_DMABUF_TESTS "Build DMA buffer tests. Relevant only if HAILO_BUILD_UT is ON" OFF)
 option(HAILO_BUILD_HW_DEBUG_TOOL "Build hw debug tool" OFF)
 option(HAILO_BUILD_GSTREAMER "Compile gstreamer plugins" OFF)
 option(HAILO_BUILD_EXAMPLES "Build examples" OFF)
@@ -30,8 +31,8 @@ endif()
 
 # Set firmware version
 add_definitions( -DFIRMWARE_VERSION_MAJOR=4 )
-add_definitions( -DFIRMWARE_VERSION_MINOR=16 )
+add_definitions( -DFIRMWARE_VERSION_MINOR=17 )
-add_definitions( -DFIRMWARE_VERSION_REVISION=2 )
+add_definitions( -DFIRMWARE_VERSION_REVISION=0 )
 if(HAILO_BUILD_SERVICE)
     add_definitions( -DHAILO_SUPPORT_MULTI_PROCESS )
 endif()

hailort/LICENSE-3RD-PARTY.md

@@ -1,16 +1,17 @@
-| Package                          | Copyright (c)                     | License            | Version        | Notes                                         | References                                                                    |
+| Package                          | Copyright (c)                     | License                    | Version        | Notes                                         | References                                                                    |
-|:---------------------------------|:----------------------------------|:-------------------|:---------------|:----------------------------------------------|:------------------------------------------------------------------------------|
+|:---------------------------------|:----------------------------------|:---------------------------|:---------------|:----------------------------------------------|:------------------------------------------------------------------------------|
-| CLI11                            | University of Cincinnati          | 3-Clause BSD       | 2.2.0          | Fork                                          | https://github.com/hailo-ai/CLI11                                             |
+| CLI11                            | University of Cincinnati          | 3-Clause BSD               | 2.2.0          | Fork                                          | https://github.com/hailo-ai/CLI11                                             |
-| Catch2                           | Catch2 Authors                    | BSL-1.0            | 2.13.7         | Cloned entire package                         | https://github.com/catchorg/Catch2                                            |
+| Catch2                           | Catch2 Authors                    | BSL-1.0                    | 2.13.7         | Cloned entire package                         | https://github.com/catchorg/Catch2                                            |
-| protobuf                         | Google Inc.                       | BSD                | 21.12          | Cloned entire package                         | https://github.com/protocolbuffers/protobuf                                   |
+| protobuf                         | Google Inc.                       | BSD                        | 21.12          | Cloned entire package                         | https://github.com/protocolbuffers/protobuf                                   |
-| pybind11                         | Wenzel Jakob                      | BSD                | 2.10.1         | Cloned entire package                         | https://github.com/pybind/pybind11                                            |
+| pybind11                         | Wenzel Jakob                      | BSD                        | 2.10.1         | Cloned entire package                         | https://github.com/pybind/pybind11                                            |
-| spdlog                           | Gabi Melman                       | MIT                | 1.6.1          | Cloned entire package                         | https://github.com/gabime/spdlog                                              |
+| spdlog                           | Gabi Melman                       | MIT                        | 1.6.1          | Cloned entire package                         | https://github.com/gabime/spdlog                                              |
-| folly                            | Facebook, Inc. and its affiliates | Apache License 2.0 | v2020.08.17.00 | Copied only the file `folly/TokenBucket.h`    | https://github.com/facebook/folly                                             |
+| folly                            | Facebook, Inc. and its affiliates | Apache License 2.0         | v2020.08.17.00 | Copied only the file `folly/TokenBucket.h`    | https://github.com/facebook/folly                                             |
-| nlohmann_json_cmake_fetchcontent | ArthurSonzogni                    | MIT License        | v3.9.1         | Cloned entire package                         | https://github.com/ArthurSonzogni/nlohmann_json_cmake_fetchcontent            |
+| nlohmann_json_cmake_fetchcontent | ArthurSonzogni                    | MIT License                | v3.9.1         | Cloned entire package                         | https://github.com/ArthurSonzogni/nlohmann_json_cmake_fetchcontent            |
-| readerwriterqueue                | Cameron Desrochers                | Simplified BSD     | 1.0.3          | Cloned entire package                         | https://github.com/cameron314/readerwriterqueue                               |
+| readerwriterqueue                | Cameron Desrochers                | Simplified BSD             | 1.0.3          | Cloned entire package                         | https://github.com/cameron314/readerwriterqueue                               |
-| DotWriter                        | John Vilk                         | MIT License        | master         | Fork                                          | https://github.com/hailo-ai/DotWriter                                         |
+| DotWriter                        | John Vilk                         | MIT License                | master         | Fork                                          | https://github.com/hailo-ai/DotWriter                                         |
-| benchmark                        | Google Inc.                       | Apache License 2.0 | 1.6.0          | Cloned entire package                         | https://github.com/google/benchmark.git                                       |
+| benchmark                        | Google Inc.                       | Apache License 2.0         | 1.6.0          | Cloned entire package                         | https://github.com/google/benchmark.git                                       |
-| md5                              | Alexander Peslyak                 | cut-down BSD       | -              | Copied code from website                      | http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 |
+| md5                              | Alexander Peslyak                 | cut-down BSD               | -              | Copied code from website                      | http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 |
-| pevents                          | Mahmoud Al-Qudsi                  | MIT License        | master         | Cloned entire package                         | https://github.com/neosmart/pevents.git                                       |
+| pevents                          | Mahmoud Al-Qudsi                  | MIT License                | master         | Cloned entire package                         | https://github.com/neosmart/pevents.git                                       |
-| grpc                             | Google Inc.                       | Apache License 2.0 | 1.46.3         | Cloned entire package                         | https://github.com/grpc/grpc                                                  |
+| grpc                             | Google Inc.                       | Apache License 2.0         | 1.46.3         | Cloned entire package                         | https://github.com/grpc/grpc                                                  |
-| stb                              | Sean Barrett                      | MIT License        | 0.97           | Copied only the file `stb/stb_image_resize.h` | https://github.com/nothings/stb                                               |
+| stb                              | Sean Barrett                      | MIT License                | 0.97           | Copied only the file `stb/stb_image_resize.h` | https://github.com/nothings/stb                                               |
+| eigen                            |                                   | Mozilla Public License 2.0 | 3.4.0          | Cloned entire package                         | https://gitlab.com/libeigen/eigen                                             |

hailort/cmake/external/eigen.cmake

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 3.11.0)
+
+include(FetchContent)
+
+FetchContent_Declare(
+    eigen
+    GIT_REPOSITORY https://gitlab.com/libeigen/eigen
+    GIT_TAG 3147391d946bb4b6c68edd901f2add6ac1f31f8c # Version 3.4.0
+    GIT_SHALLOW TRUE
+    SOURCE_DIR ${HAILO_EXTERNAL_DIR}/eigen-src
+    SUBBUILD_DIR ${HAILO_EXTERNAL_DIR}/eigen-subbuild
+)
+
+
+# https://stackoverflow.com/questions/65527126/disable-install-for-fetchcontent
+FetchContent_GetProperties(eigen)
+if(NOT eigen_POPULATED)
+    FetchContent_Populate(eigen)
+    option(EIGEN_BUILD_DOC OFF)
+    option(BUILD_TESTING OFF)
+    option(EIGEN_LEAVE_TEST_IN_ALL_TARGET OFF)
+    option(EIGEN_BUILD_PKGCONFIG OFF)
+    option(CMAKE_Fortran_COMPILER OFF)
+
+    if (NOT HAILO_EXTERNALS_EXCLUDE_TARGETS)
+        add_subdirectory(${eigen_SOURCE_DIR} ${eigen_BINARY_DIR} EXCLUDE_FROM_ALL)
+    endif()
+endif()

hailort/common/barrier.cpp

@@ -36,6 +36,9 @@ void Barrier::arrive_and_wait()
 void Barrier::terminate()
 {
     m_is_activated.store(false);
+    {
+        std::unique_lock<std::mutex> lock(m_mutex);
+    }
     m_cv.notify_all();
 }
 

hailort/common/device_measurements.cpp

@@ -134,7 +134,7 @@ hailo_status PowerMeasurement::start_measurement()
     CHECK_SUCCESS(status, "Failed to start power measurement");
    
     m_is_thread_running = true;
-    m_thread = std::thread([this] () {
+    m_thread = std::thread([this] () -> hailo_status {
         const bool clear_power_measurement_history = true;
         while (m_is_thread_running.load()) { 
             std::this_thread::sleep_for(DEFAULT_MEASUREMENTS_INTERVAL);

hailort/common/os/posix/os_utils.cpp

@@ -100,7 +100,7 @@ size_t OsUtils::get_dma_able_alignment()
 // TODO: implement on qnx (HRT-12356) - only needed when async api is implemented on qnx
 // TODO - URT-13534 - use sys call for QNX OS to get page size
 #elif defined(__QNX__)
-    return OS_UTILS__QNX_PAGE_SIZE
+    return OS_UTILS__QNX_PAGE_SIZE;
 #endif
 }
 

hailort/common/os/posix/socket.cpp

@@ -230,7 +230,7 @@ hailo_status Socket::send_to(const uint8_t *src_buffer, size_t src_buffer_size,
         } else if (EPIPE == errno) {
             // When socket is aborted from another thread sendto will return errno EPIPE
             LOGGER__INFO("Udp send aborted!");
-            return HAILO_STREAM_ABORTED_BY_USER;
+            return HAILO_STREAM_ABORT;
         } else {
             LOGGER__ERROR("Udp failed to send data, errno:{}.", errno);
             return HAILO_ETH_SEND_FAILURE;
@@ -272,7 +272,7 @@ hailo_status Socket::recv_from(uint8_t *dest_buffer, size_t dest_buffer_size, in
     }
     else if ((0 == number_of_received_bytes) && (0 != dest_buffer_size)) {
         LOGGER__INFO("Udp socket was aborted");
-        return HAILO_STREAM_ABORTED_BY_USER;
+        return HAILO_STREAM_ABORT;
     }
 
     if (result_src_addr_size > src_addr_size) {

hailort/common/runtime_statistics_internal.hpp

@@ -15,10 +15,69 @@
 #include <cmath>
 #include <mutex>
 #include <limits>
+#include <ostream>
+#include <sstream>
+#include <iomanip>
 
 namespace hailort
 {
 
+class AccumulatorResultsHelper final
+{
+public:
+    AccumulatorResultsHelper() = delete;
+
+    static const uint32_t DEFAULT_FLOATING_POINT_PRECISION = 4;
+
+    static std::string format_results(const AccumulatorResults &results, bool verbose = false,
+        uint32_t precision = DEFAULT_FLOATING_POINT_PRECISION)
+    {
+        std::stringstream stream;
+        stream << format_statistic(results.count(), "count") << ", ";
+        stream << format_statistic(results.mean(), "mean", precision);
+        if (verbose) {
+            stream << ", ";
+            stream << format_statistic(results.min(), "min", precision) << ", ";
+            stream << format_statistic(results.max(), "max", precision) << ", ";
+            stream << format_statistic(results.var(), "var", precision) << ", ";
+            stream << format_statistic(results.sd(), "sd", precision) << ", ";
+            stream << format_statistic(results.mean_sd(), "mean_sd", precision);
+        }
+        return stream.str();
+    }
+
+    static std::string format_statistic(const Expected<double> &statistic, const std::string &name = "",
+        uint32_t precision = DEFAULT_FLOATING_POINT_PRECISION)
+    {
+        return format_statistic<double>(statistic, name, precision);
+    }
+
+    static std::string format_statistic(const Expected<size_t> &statistic, const std::string &name = "")
+    {
+        return format_statistic<size_t>(statistic, name);
+    }
+
+private:
+    template<typename T, std::enable_if_t<std::is_arithmetic<T>::value, int> = 0>
+    static std::string format_statistic(const Expected<T> &statistic, const std::string &name,
+        uint32_t precision = DEFAULT_FLOATING_POINT_PRECISION)
+    {
+        static const std::string NO_VALUE = "-";
+        std::stringstream stream;
+        if (!name.empty()) {
+            stream << name << "=";
+        }
+
+        if (statistic.has_value()) {
+            stream << std::fixed << std::setprecision(precision) << statistic.value();
+        } else {
+            stream << NO_VALUE;
+        }
+
+        return stream.str();
+    }
+};
+
 template<typename T, std::enable_if_t<std::is_arithmetic<T>::value, int> = 0>
 class FullAccumulator : public Accumulator<T>
 {

hailort/common/utils.hpp

@@ -12,10 +12,13 @@
 #ifndef HAILO_UTILS_H_
 #define HAILO_UTILS_H_
 
-#include <assert.h>
+#include "hailo/hailort.h"
-#include <hailo/hailort.h>
+#include "hailo/expected.hpp"
+
 #include "common/logger_macros.hpp"
 #include <spdlog/fmt/bundled/core.h>
+
+#include <assert.h>
 #include <map>
 #include <set>
 #include <unordered_set>
@@ -166,6 +169,17 @@ _ISEMPTY(                                                               \
 #define CONSTRUCT_MSG(dft_fmt, ...) _CONSTRUCT_MSG(ISEMPTY(__VA_ARGS__), dft_fmt, "" __VA_ARGS__)
 
 
+inline hailo_status get_status(hailo_status status)
+{
+    return status;
+}
+
+template<typename T>
+inline hailo_status get_status(const Expected<T> &exp)
+{
+    return exp.status();
+}
+
 #define _CHECK(cond, ret_val, ...)      \
     do {                                \
         if (!(cond)) {                  \
@@ -175,39 +189,31 @@ _ISEMPTY(                                                               \
     } while(0)
 
 /** Returns ret_val when cond is false */
-#define CHECK(cond, ret_val, ...) _CHECK((cond), (ret_val), CONSTRUCT_MSG("CHECK failed", ##__VA_ARGS__))
+#define CHECK(cond, ret_val, ...) \
-#define CHECK_AS_EXPECTED(cond, ret_val, ...) \
+    _CHECK((cond), make_unexpected(ret_val), CONSTRUCT_MSG("CHECK failed", ##__VA_ARGS__))
-    _CHECK((cond), (make_unexpected(ret_val)), CONSTRUCT_MSG("CHECK_AS_EXPECTED failed", ##__VA_ARGS__))
+#define CHECK_AS_EXPECTED CHECK
 
-#define CHECK_ARG_NOT_NULL(arg) _CHECK(nullptr != (arg), HAILO_INVALID_ARGUMENT, "CHECK_ARG_NOT_NULL for {} failed", #arg)
+#define CHECK_ARG_NOT_NULL(arg) _CHECK(nullptr != (arg), make_unexpected(HAILO_INVALID_ARGUMENT), "CHECK_ARG_NOT_NULL for {} failed", #arg)
+#define CHECK_ARG_NOT_NULL_AS_EXPECTED CHECK_ARG_NOT_NULL
 
-#define CHECK_ARG_NOT_NULL_AS_EXPECTED(arg) _CHECK(nullptr != (arg), make_unexpected(HAILO_INVALID_ARGUMENT), "CHECK_ARG_NOT_NULL_AS_EXPECTED for {} failed", #arg)
+#define CHECK_NOT_NULL(arg, status) _CHECK(nullptr != (arg), make_unexpected(status), "CHECK_NOT_NULL for {} failed", #arg)
+#define CHECK_NOT_NULL_AS_EXPECTED CHECK_NOT_NULL
 
-#define CHECK_NOT_NULL(arg, status) _CHECK(nullptr != (arg), status, "CHECK_NOT_NULL for {} failed", #arg)
+#define _CHECK_SUCCESS(res, is_default, fmt, ...)                                                                               \
-
-#define CHECK_NOT_NULL_AS_EXPECTED(arg, status) _CHECK(nullptr != (arg), make_unexpected(status), "CHECK_NOT_NULL_AS_EXPECTED for {} failed", #arg)
-
-#define _CHECK_SUCCESS(status, is_default, fmt, ...)                                                                            \
     do {                                                                                                                        \
-        const auto &__check_success_status = (status);                                                                          \
+        const auto &__check_success_status = get_status(res);                                                                   \
         _CHECK(                                                                                                                 \
-            HAILO_SUCCESS == __check_success_status,                                                                            \
+            (HAILO_SUCCESS == __check_success_status),                                                                          \
-            __check_success_status,                                                                                             \
+            make_unexpected(__check_success_status),                                                                            \
             _CONSTRUCT_MSG(is_default, "CHECK_SUCCESS failed with status={}", fmt, __check_success_status, ##__VA_ARGS__)       \
         );                                                                                                                      \
     } while(0)
 #define CHECK_SUCCESS(status, ...) _CHECK_SUCCESS(status, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__)
+#define CHECK_SUCCESS_AS_EXPECTED CHECK_SUCCESS
 
-#define _CHECK_SUCCESS_AS_EXPECTED(status, is_default, fmt, ...)                                                                       \
+#define _CHECK_EXPECTED _CHECK_SUCCESS
-    do {                                                                                                                               \
+#define CHECK_EXPECTED(obj, ...) _CHECK_EXPECTED(obj, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__)
-        const auto &__check_success_status = (status);                                                                                 \
+#define CHECK_EXPECTED_AS_STATUS CHECK_EXPECTED
-        _CHECK(                                                                                                                        \
-            HAILO_SUCCESS == __check_success_status,                                                                                   \
-            make_unexpected(__check_success_status),                                                                                   \
-            _CONSTRUCT_MSG(is_default, "CHECK_SUCCESS_AS_EXPECTED failed with status={}", fmt, __check_success_status, ##__VA_ARGS__)  \
-        );                                                                                                                             \
-    } while(0)
-#define CHECK_SUCCESS_AS_EXPECTED(status, ...) _CHECK_SUCCESS_AS_EXPECTED(status, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__)
 
 // Define macro CHECK_IN_DEBUG - that checks cond in debug with CHECK macro but in release does nothing and will get optimized out
 #ifdef NDEBUG
@@ -258,28 +264,30 @@ _ISEMPTY(                                                               \
 #define CHECK_GRPC_STATUS_AS_EXPECTED(status) _CHECK_GRPC_STATUS(status, make_unexpected(HAILO_RPC_FAILED), SERVICE_WARNING_MSG)
 #endif
 
-#define _CHECK_EXPECTED(obj, is_default, fmt, ...)                                                                                      \
+#define __HAILO_CONCAT(x, y) x ## y
-    do {                                                                                                                                \
+#define _HAILO_CONCAT(x, y) __HAILO_CONCAT(x, y)
-        const auto &__check_expected_obj = (obj);                                                                                       \
-        _CHECK(                                                                                                                         \
-            __check_expected_obj.has_value(),                                                                                           \
-            make_unexpected(__check_expected_obj.status()),                                                                             \
-            _CONSTRUCT_MSG(is_default, "CHECK_EXPECTED failed with status={}", fmt, __check_expected_obj.status(), ##__VA_ARGS__)       \
-        );                                                                                                                              \
-    } while(0)
-#define CHECK_EXPECTED(obj, ...) _CHECK_EXPECTED(obj, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__)
 
+#define _TRY(expected_var_name, var_decl, expr, ...) \
+    auto expected_var_name = (expr); \
+    CHECK_EXPECTED(expected_var_name, __VA_ARGS__); \
+    var_decl = expected_var_name.release()
 
-#define _CHECK_EXPECTED_AS_STATUS(obj, is_default, fmt, ...)                                                                                      \
+/**
-    do {                                                                                                                                          \
+ * The TRY macro is used to allow easier validation and access for variables returned as Expected<T>.
-        const auto &__check_expected_obj = (obj);                                                                                                 \
+ * If the expression returns an Expected<T> with status HAILO_SUCCESS, the macro will release the expected and assign
-        _CHECK(                                                                                                                                   \
+ * the var_decl.
-            __check_expected_obj.has_value(),                                                                                                     \
+ * Otherwise, the macro will cause current function to return the failed status.
-            __check_expected_obj.status(),                                                                                                        \
+ *
-            _CONSTRUCT_MSG(is_default, "CHECK_EXPECTED_AS_STATUS failed with status={}", fmt, __check_expected_obj.status(), ##__VA_ARGS__)       \
+ * Usage example:
-        );                                                                                                                                        \
+ *
-    } while(0)
+ * Expected<int> func() {
-#define CHECK_EXPECTED_AS_STATUS(obj, ...) _CHECK_EXPECTED_AS_STATUS(obj, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__)
+ *     TRY(auto var, return_5());
+ *     // Now var is int with value 5
+ *
+ *     // func will return Unexpected with status HAILO_INTERNAL_FAILURE
+ *     TRY(auto var2, return_error(HAILO_INTERNAL_FAILURE), "Failed doing stuff {}", 5);
+ */
+#define TRY(var_decl, expr, ...) _TRY(_HAILO_CONCAT(__expected, __COUNTER__), var_decl, expr, __VA_ARGS__)
 
 #ifndef _MSC_VER
 #define IGNORE_DEPRECATION_WARNINGS_BEGIN _Pragma("GCC diagnostic push") \

hailort/drivers/common/hailo_ioctl_common.h

@@ -14,7 +14,10 @@
 #define SIZE_OF_VDMA_DESCRIPTOR         (16)
 #define VDMA_DEST_CHANNELS_START        (16)
 
-#define CHANNEL_IRQ_TIMESTAMPS_SIZE (128 * 2) // Should be same as MAX_IRQ_TIMESTAMPS_SIZE (hailort_driver.hpp)
+#define HAILO_VDMA_MAX_ONGOING_TRANSFERS (128)
+#define HAILO_VDMA_MAX_ONGOING_TRANSFERS_MASK (HAILO_VDMA_MAX_ONGOING_TRANSFERS - 1)
+
+#define CHANNEL_IRQ_TIMESTAMPS_SIZE (HAILO_VDMA_MAX_ONGOING_TRANSFERS * 2)
 #define CHANNEL_IRQ_TIMESTAMPS_SIZE_MASK (CHANNEL_IRQ_TIMESTAMPS_SIZE - 1)
 
 #define INVALID_DRIVER_HANDLE_VALUE     ((uintptr_t)-1)
@@ -35,14 +38,13 @@ typedef ULONG uint32_t;
 typedef UCHAR uint8_t;
 typedef USHORT uint16_t;
 typedef ULONGLONG uint64_t;
-typedef uint64_t u64;
-typedef uint32_t u32;
-typedef uint16_t u16;
-typedef uint8_t  u8;
 #endif /*  !defined(__cplusplus) && defined(NTDDI_VERSION) */
 
 
 #ifdef _MSC_VER
+
+#include <initguid.h>
+
 #if !defined(bool) && !defined(__cplusplus)
 typedef uint8_t bool;
 #endif // !defined(bool) && !defined(__cplusplus)
@@ -51,6 +53,48 @@ typedef uint8_t bool;
 #define INT_MAX 0x7FFFFFFF
 #endif // !defined(INT_MAX)
 
+
+// {d88d31f1-fede-4e71-ac2a-6ce0018c1501}
+DEFINE_GUID (GUID_DEVINTERFACE_HailoKM,
+    0xd88d31f1,0xfede,0x4e71,0xac,0x2a,0x6c,0xe0,0x01,0x8c,0x15,0x01);
+
+#define HAILO_GENERAL_IOCTL_MAGIC 0
+#define HAILO_VDMA_IOCTL_MAGIC    1
+#define HAILO_NON_LINUX_IOCTL_MAGIC 2
+
+#define HAILO_IOCTL_COMPATIBLE                  CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_ANY_ACCESS)
+
+
+typedef struct tCompatibleHailoIoctlParam
+{
+    union {
+        struct {
+            ULONG Size : 16;
+            ULONG Code : 8;
+            ULONG Type : 6;
+            ULONG Read : 1;
+            ULONG Write : 1;
+        } bits;
+        ULONG value;
+    } u;
+} tCompatibleHailoIoctlParam;
+
+static ULONG FORCEINLINE _IOC_(ULONG nr, ULONG type, ULONG size, bool read, bool write)
+{
+    struct tCompatibleHailoIoctlParam param;
+    param.u.bits.Code = nr;
+    param.u.bits.Size = size;
+    param.u.bits.Type = type;
+    param.u.bits.Read = read ? 1 : 0;
+    param.u.bits.Write = write ? 1 : 0;
+    return param.u.value;
+}
+
+#define _IOW_(type,nr,size) _IOC_(nr, type, sizeof(size), true, false)
+#define _IOR_(type,nr,size) _IOC_(nr, type, sizeof(size), false, true)
+#define _IOWR_(type,nr,size) _IOC_(nr, type, sizeof(size), true, true)
+#define _IO_(type,nr) _IOC_(nr, type, 0, false, false)
+
 #elif defined(__linux__) // #ifdef _MSC_VER
 #ifndef __KERNEL__
 // include the userspace headers only if this file is included by user space program
@@ -149,11 +193,17 @@ struct hailo_vdma_buffer_unmap_params {
 /* structure used in ioctl HAILO_DESC_LIST_CREATE */
 struct hailo_desc_list_create_params {
     size_t desc_count;          // in
+    uint16_t desc_page_size;    // in
     bool is_circular;           // in
     uintptr_t desc_handle;      // out
     uint64_t dma_address;       // out
 };
 
+/* structure used in ioctl HAILO_DESC_LIST_RELEASE */
+struct hailo_desc_list_release_params {
+    uintptr_t desc_handle;      // in
+};
+
 /* structure used in ioctl HAILO_NON_LINUX_DESC_LIST_MMAP */
 struct hailo_non_linux_desc_list_mmap_params {
     uintptr_t desc_handle;  // in
@@ -164,8 +214,9 @@ struct hailo_non_linux_desc_list_mmap_params {
 /* structure used in ioctl HAILO_DESC_LIST_BIND_VDMA_BUFFER */
 struct hailo_desc_list_bind_vdma_buffer_params {
     size_t buffer_handle;       // in
+    size_t buffer_size;         // in
+    size_t buffer_offset;       // in
     uintptr_t desc_handle;      // in
-    uint16_t desc_page_size;    // in
     uint8_t channel_index;      // in
     uint32_t starting_desc;     // in
 };
@@ -189,6 +240,7 @@ struct hailo_vdma_interrupts_channel_data {
     uint16_t host_num_processed;
     uint8_t host_error;             // Channel errors bits on source side
     uint8_t device_error;           // Channel errors bits on dest side
+    bool validation_success;        // If the validation of the channel was successful
 };
 
 struct hailo_vdma_interrupts_wait_params {
@@ -272,26 +324,6 @@ struct hailo_memory_transfer_params {
     uint8_t buffer[MAX_MEMORY_TRANSFER_LENGTH];         // in/out
 };
 
-/* structure used in ioctl HAILO_VDMA_CHANNEL_READ_REGISTER */
-struct hailo_vdma_channel_read_register_params {
-    uint8_t engine_index;                       // in
-    uint8_t channel_index;                      // in
-    enum hailo_dma_data_direction direction;    // in
-    size_t offset;                              // in
-    size_t reg_size;                            // in, can be either 1, 2 or 4
-    uint32_t data;                              // out
-};
-
-/* structure used in ioctl HAILO_VDMA_CHANNEL_WRITE_REGISTER */
-struct hailo_vdma_channel_write_register_params {
-    uint8_t engine_index;                       // in
-    uint8_t channel_index;                      // in
-    enum hailo_dma_data_direction direction;    // in
-    size_t offset;                              // in
-    size_t reg_size;                            // in, can be either 1, 2 or 4
-    uint32_t data;                              // in
-};
-
 /* structure used in ioctl HAILO_VDMA_BUFFER_SYNC */
 enum hailo_vdma_buffer_sync_type {
     HAILO_SYNC_FOR_CPU,
@@ -362,21 +394,103 @@ struct hailo_read_log_params {
     size_t read_bytes;                          // out
 };
 
+/* structure used in ioctl HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC */
 struct hailo_allocate_low_memory_buffer_params {
     size_t      buffer_size;    // in
     uintptr_t   buffer_handle;  // out
 };
 
+/* structure used in ioctl HAILO_VDMA_LOW_MEMORY_BUFFER_FREE */
+struct hailo_free_low_memory_buffer_params {
+    uintptr_t  buffer_handle;  // in
+};
+
 struct hailo_mark_as_in_use_params {
     bool in_use;           // out
 };
 
+/* structure used in ioctl HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC */
 struct hailo_allocate_continuous_buffer_params {
     size_t buffer_size;         // in
     uintptr_t buffer_handle;    // out
     uint64_t dma_address;       // out
 };
 
+/* structure used in ioctl HAILO_VDMA_CONTINUOUS_BUFFER_FREE */
+struct hailo_free_continuous_buffer_params {
+    uintptr_t buffer_handle;    // in
+};
+
+/* structures used in ioctl HAILO_VDMA_LAUNCH_TRANSFER */
+struct hailo_vdma_transfer_buffer {
+    size_t mapped_buffer_handle;       // in
+    uint32_t offset;                   // in
+    uint32_t size;                     // in
+};
+
+enum hailo_vdma_interrupts_domain {
+    HAILO_VDMA_INTERRUPTS_DOMAIN_NONE   = 0,
+    HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE = (1 << 0),
+    HAILO_VDMA_INTERRUPTS_DOMAIN_HOST   = (1 << 1),
+
+    /** Max enum value to maintain ABI Integrity */
+    HAILO_VDMA_INTERRUPTS_DOMAIN_MAX_ENUM = INT_MAX,
+};
+
+// We allow maximum 2 buffers per transfer since we may have an extra buffer 
+// to make sure each buffer is aligned to page size.
+#define HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER (2)
+
+struct hailo_vdma_launch_transfer_params {
+    uint8_t engine_index;                                               // in
+    uint8_t channel_index;                                              // in
+
+    uintptr_t desc_handle;                                              // in
+    uint32_t starting_desc;                                             // in
+
+    bool should_bind;                                                   // in, if false, assumes buffer already bound.
+    uint8_t buffers_count;                                              // in
+    struct hailo_vdma_transfer_buffer
+        buffers[HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER];                 // in
+
+    enum hailo_vdma_interrupts_domain first_interrupts_domain;          // in
+    enum hailo_vdma_interrupts_domain last_interrupts_domain;           // in
+
+    bool is_debug;                                                      // in, if set program hw to send
+                                                                        // more info (e.g desc complete status)
+
+    uint32_t descs_programed;                                           // out, amount of descriptors programed.
+};
+
+#ifdef _MSC_VER
+struct tCompatibleHailoIoctlData
+{
+    tCompatibleHailoIoctlParam Parameters;
+    ULONG_PTR Value;
+    union {
+        struct hailo_memory_transfer_params MemoryTransfer;
+        struct hailo_vdma_interrupts_enable_params VdmaInterruptsEnable;
+        struct hailo_vdma_interrupts_disable_params VdmaInterruptsDisable;
+        struct hailo_vdma_interrupts_read_timestamp_params VdmaInterruptsReadTimestamps;
+        struct hailo_vdma_interrupts_wait_params VdmaInterruptsWait;
+        struct hailo_vdma_buffer_sync_params VdmaBufferSync;
+        struct hailo_fw_control FirmwareControl;
+        struct hailo_vdma_buffer_map_params VdmaBufferMap;
+        struct hailo_vdma_buffer_unmap_params VdmaBufferUnmap;
+        struct hailo_desc_list_create_params DescListCreate;
+        struct hailo_desc_list_release_params DescListReleaseParam;
+        struct hailo_desc_list_bind_vdma_buffer_params DescListBind;
+        struct hailo_d2h_notification D2HNotification;
+        struct hailo_device_properties DeviceProperties;
+        struct hailo_driver_info DriverInfo;
+        struct hailo_non_linux_desc_list_mmap_params DescListMmap;
+        struct hailo_read_log_params ReadLog;
+        struct hailo_mark_as_in_use_params MarkAsInUse;
+        struct hailo_vdma_launch_transfer_params LaunchTransfer;
+    } Buffer;
+};
+#endif // _MSC_VER
+
 #pragma pack(pop)
 
 enum hailo_general_ioctl_code {
@@ -407,8 +521,6 @@ enum hailo_vdma_ioctl_code {
     HAILO_VDMA_INTERRUPTS_DISABLE_CODE,
     HAILO_VDMA_INTERRUPTS_WAIT_CODE,
     HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE,
-    HAILO_VDMA_CHANNEL_READ_REGISTER_CODE,
-    HAILO_VDMA_CHANNEL_WRITE_REGISTER_CODE,
     HAILO_VDMA_BUFFER_MAP_CODE,
     HAILO_VDMA_BUFFER_UNMAP_CODE,
     HAILO_VDMA_BUFFER_SYNC_CODE,
@@ -420,6 +532,7 @@ enum hailo_vdma_ioctl_code {
     HAILO_MARK_AS_IN_USE_CODE,
     HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE,
     HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE,
+    HAILO_VDMA_LAUNCH_TRANSFER_CODE,
 
     // Must be last
     HAILO_VDMA_IOCTL_MAX_NR,
@@ -430,24 +543,23 @@ enum hailo_vdma_ioctl_code {
 #define HAILO_VDMA_INTERRUPTS_WAIT            _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_WAIT_CODE,              struct hailo_vdma_interrupts_wait_params)
 #define HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE,   struct hailo_vdma_interrupts_read_timestamp_params)
 
-#define HAILO_VDMA_CHANNEL_READ_REGISTER      _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_READ_REGISTER_CODE,         struct hailo_vdma_channel_read_register_params)
-#define HAILO_VDMA_CHANNEL_WRITE_REGISTER     _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_CHANNEL_WRITE_REGISTER_CODE,        struct hailo_vdma_channel_write_register_params)
-
 #define HAILO_VDMA_BUFFER_MAP                 _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE,                    struct hailo_vdma_buffer_map_params)
 #define HAILO_VDMA_BUFFER_UNMAP               _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_BUFFER_UNMAP_CODE,                  struct hailo_vdma_buffer_unmap_params)
 #define HAILO_VDMA_BUFFER_SYNC                _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_BUFFER_SYNC_CODE,                   struct hailo_vdma_buffer_sync_params)
 
 #define HAILO_DESC_LIST_CREATE                _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE,                   struct hailo_desc_list_create_params)
-#define HAILO_DESC_LIST_RELEASE               _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_DESC_LIST_RELEASE_CODE,                  uintptr_t)
+#define HAILO_DESC_LIST_RELEASE               _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_DESC_LIST_RELEASE_CODE,                  struct hailo_desc_list_release_params)
 #define HAILO_DESC_LIST_BIND_VDMA_BUFFER      _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_DESC_LIST_BIND_VDMA_BUFFER_CODE,         struct hailo_desc_list_bind_vdma_buffer_params)
 
 #define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC    _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE,       struct hailo_allocate_low_memory_buffer_params)
-#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE     _IO_(HAILO_VDMA_IOCTL_MAGIC,   HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE)
+#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE     _IOR_(HAILO_VDMA_IOCTL_MAGIC,   HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE,       struct hailo_free_low_memory_buffer_params)
 
 #define HAILO_MARK_AS_IN_USE                  _IOW_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_MARK_AS_IN_USE_CODE,                     struct hailo_mark_as_in_use_params)
 
 #define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC    _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE,       struct hailo_allocate_continuous_buffer_params)
-#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE     _IO_(HAILO_VDMA_IOCTL_MAGIC,   HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE)
+#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE     _IOR_(HAILO_VDMA_IOCTL_MAGIC,   HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE,       struct hailo_free_continuous_buffer_params)
+
+#define HAILO_VDMA_LAUNCH_TRANSFER           _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LAUNCH_TRANSFER_CODE,                struct hailo_vdma_launch_transfer_params)
 
 
 enum hailo_non_linux_ioctl_code {

hailort/drivers/win/include/Public.h

@@ -15,13 +15,9 @@ Environment:
 
 --*/
 
-//
+#ifndef _HAILO_PUBLIC_H_
-// Define an Interface Guid so that apps can find the device and talk to it.
+#define _HAILO_PUBLIC_H_
-//
 
-DEFINE_GUID (GUID_DEVINTERFACE_HailoKM,
-    0xd88d31f1,0xfede,0x4e71,0xac,0x2a,0x6c,0xe0,0x01,0x8c,0x15,0x01);
-// {d88d31f1-fede-4e71-ac2a-6ce0018c1501}
 
 #define HAILO_IOCTL_COMMON                      CTL_CODE(FILE_DEVICE_UNKNOWN, 0x801, METHOD_BUFFERED, FILE_ANY_ACCESS)
 #define IOCTL_FUNC(x)                           (((x) >> 2) & 0xfff)
@@ -57,69 +53,7 @@ struct tCommonHailoIoctlParam
 #define HAILO_CMD_FREE_MEMORY           0x0060
 #define HAILO_CMD_ALLOC_MEMORY          0x0061
 
-#define HAILO_IOCTL_COMPATIBLE                  CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_ANY_ACCESS)
-struct tCompatibleHailoIoctlParam
-{
-    union {
-        struct {
-            ULONG Size : 16;
-            ULONG Code : 8;
-            ULONG Type : 6;
-            ULONG Read : 1;
-            ULONG Write : 1;
-        } bits;
-        ULONG value;
-    } u;
-};
-
-#define HAILO_GENERAL_IOCTL_MAGIC 0
-#define HAILO_VDMA_IOCTL_MAGIC    1
-#define HAILO_NON_LINUX_IOCTL_MAGIC 2
-
-
-
-static ULONG FORCEINLINE _IOC_(ULONG nr, ULONG type, ULONG size, bool read, bool write)
-{
-    tCompatibleHailoIoctlParam param;
-    param.u.bits.Code = nr;
-    param.u.bits.Size = size;
-    param.u.bits.Type = type;
-    param.u.bits.Read = read ? 1 : 0;
-    param.u.bits.Write = write ? 1 : 0;
-    return param.u.value;
-}
-
-#define _IOW_(type,nr,size) _IOC_(nr, type, sizeof(size), true, false)
-#define _IOR_(type,nr,size) _IOC_(nr, type, sizeof(size), false, true)
-#define _IOWR_(type,nr,size) _IOC_(nr, type, sizeof(size), true, true)
-#define _IO_(type,nr) _IOC_(nr, type, 0, false, false)
 
 #include "..\..\common\hailo_ioctl_common.h"
 
-struct tCompatibleHailoIoctlData
+#endif /* _HAILO_PUBLIC_H_ */
-{
-    tCompatibleHailoIoctlParam Parameters;
-    ULONG_PTR Value;
-    union {
-        hailo_memory_transfer_params MemoryTransfer;
-        hailo_vdma_interrupts_enable_params VdmaInterruptsEnable;
-        hailo_vdma_interrupts_disable_params VdmaInterruptsDisable;
-        hailo_vdma_interrupts_read_timestamp_params VdmaInterruptsReadTimestamps;
-        hailo_vdma_interrupts_wait_params VdmaInterruptsWait;
-        hailo_vdma_buffer_sync_params VdmaBufferSync;
-        hailo_fw_control FirmwareControl;
-        hailo_vdma_buffer_map_params VdmaBufferMap;
-        hailo_vdma_buffer_unmap_params VdmaBufferUnmap;
-        hailo_desc_list_create_params DescListCreate;
-        uintptr_t DescListReleaseParam;
-        hailo_desc_list_bind_vdma_buffer_params DescListBind;
-        hailo_d2h_notification D2HNotification;
-        hailo_device_properties DeviceProperties;
-        hailo_driver_info DriverInfo;
-        hailo_vdma_channel_read_register_params ChannelRegisterRead;
-        hailo_vdma_channel_write_register_params ChannelRegisterWrite;
-        hailo_non_linux_desc_list_mmap_params DescListMmap;
-        hailo_read_log_params ReadLog;
-        hailo_mark_as_in_use_params MarkAsInUse;
-    } Buffer;
-};

hailort/hailort_service/CMakeLists.txt

@@ -12,6 +12,7 @@ endif()
 
 add_executable(hailort_service
     hailort_rpc_service.cpp
+    cng_buffer_pool.cpp
     service_resource_manager.hpp
     ${HAILORT_SERVICE_OS_DIR}/hailort_service.cpp
     ${HAILORT_COMMON_CPP_SOURCES}

hailort/hailort_service/cng_buffer_pool.cpp

@@ -0,0 +1,163 @@
+/**
+ * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file cng_buffer_pool.cpp
+ * @brief Network group buffer pool implementation
+ **/
+
+#include "cng_buffer_pool.hpp"
+#include "service_resource_manager.hpp"
+#include "hailo/hailort.h"
+
+namespace hailort
+{
+
+
+Expected<std::shared_ptr<ServiceStreamBufferPool>> ServiceStreamBufferPool::create(uint32_t vdevice_handle,
+    size_t buffer_size, size_t buffer_count, hailo_dma_buffer_direction_t direction, EventPtr shutdown_event)
+{
+    auto map_buffer_lambda = [direction](std::shared_ptr<VDevice> vdevice, BufferPtr buffer) {
+        return DmaMappedBuffer::create(*vdevice, buffer->data(), buffer->size(), direction);
+    };
+    auto &vdevice_manager = ServiceResourceManager<VDevice>::get_instance();
+
+    auto free_buffers_queue = SpscQueue<BufferPtr>::create(buffer_count, shutdown_event, DEFAULT_TRANSFER_TIMEOUT);
+    CHECK_EXPECTED(free_buffers_queue);
+
+    std::vector<AllocatedMappedBuffer> buffers;
+    buffers.reserve(buffer_count);
+    for (size_t i = 0; i < buffer_count; i++) {
+        auto buffer = Buffer::create_shared(buffer_size, BufferStorageParams::create_dma());
+        CHECK_EXPECTED(buffer);
+
+        auto mapped_buffer = vdevice_manager.execute<Expected<DmaMappedBuffer>>(vdevice_handle, map_buffer_lambda, buffer.value());
+        CHECK_EXPECTED(mapped_buffer);
+
+        auto status = free_buffers_queue->enqueue(buffer.value());
+        CHECK_SUCCESS(status);
+
+        buffers.emplace_back(AllocatedMappedBuffer{ buffer.release(), mapped_buffer.release()});
+    }
+
+    auto buffer_pool_ptr = make_shared_nothrow<ServiceStreamBufferPool>(buffer_size, std::move(buffers),
+        free_buffers_queue.release(), buffer_count);
+    CHECK_NOT_NULL_AS_EXPECTED(buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+    return buffer_pool_ptr;
+}
+
+ServiceStreamBufferPool::ServiceStreamBufferPool(size_t buffer_size, std::vector<AllocatedMappedBuffer> &&buffers,
+        SpscQueue<BufferPtr> &&free_buffers_queue, size_t buffers_count) :
+    m_buffer_size(buffer_size),
+    m_buffers_count(buffers_count),
+    m_buffers(std::move(buffers)),
+    m_free_buffers_queue(std::move(free_buffers_queue))
+{}
+
+Expected<BufferPtr> ServiceStreamBufferPool::acquire_buffer()
+{
+    auto buffer = m_free_buffers_queue.dequeue(DEFAULT_TRANSFER_TIMEOUT);
+    if (HAILO_SHUTDOWN_EVENT_SIGNALED == buffer.status()) {
+        return make_unexpected(buffer.status());
+    }
+    else if (HAILO_TIMEOUT == buffer.status()) {
+        LOGGER__WARNING(
+            "Failed to acquire buffer because the buffer pool is empty. This could be caused by uneven reading and writing speeds");
+        return make_unexpected(buffer.status());
+    }
+    CHECK_EXPECTED(buffer);
+
+    return buffer.release();
+}
+
+hailo_status ServiceStreamBufferPool::return_to_pool(BufferPtr buffer)
+{
+    CHECK(buffer->size() == m_buffer_size, HAILO_INTERNAL_FAILURE,
+        "Buffer size is not the same as expected for pool! ({} != {})", buffer->size(), m_buffer_size);
+
+    std::unique_lock<std::mutex> lock(m_mutex);
+    auto status = m_free_buffers_queue.enqueue(buffer);
+    CHECK_SUCCESS(status);
+
+    return HAILO_SUCCESS;
+}
+
+size_t ServiceStreamBufferPool::buffers_count()
+{
+    return m_buffers_count;
+}
+
+Expected<std::shared_ptr<ServiceNetworkGroupBufferPool>> ServiceNetworkGroupBufferPool::create(uint32_t vdevice_handle)
+{
+    auto shutdown_event_exp = Event::create_shared(Event::State::not_signalled);
+    CHECK_EXPECTED(shutdown_event_exp);
+    auto shutdown_event = shutdown_event_exp.release();
+
+    auto cng_buffer_pool_ptr = make_shared_nothrow<ServiceNetworkGroupBufferPool>(shutdown_event, vdevice_handle);
+    CHECK_NOT_NULL_AS_EXPECTED(cng_buffer_pool_ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+    return cng_buffer_pool_ptr;
+}
+
+ServiceNetworkGroupBufferPool::ServiceNetworkGroupBufferPool(EventPtr shutdown_event, uint32_t vdevice_handle) :
+    m_output_name_to_buffer_pool(), m_shutdown_event(shutdown_event), m_vdevice_handle(vdevice_handle)
+{}
+
+hailo_status ServiceNetworkGroupBufferPool::allocate_pool(const std::string &name, size_t frame_size, size_t pool_size)
+{
+    auto buffer_pool = ServiceStreamBufferPool::create(m_vdevice_handle, frame_size,
+        pool_size, HAILO_DMA_BUFFER_DIRECTION_D2H, m_shutdown_event);
+    CHECK_EXPECTED(buffer_pool);
+
+    std::unique_lock<std::mutex> lock(m_mutex);
+    m_output_name_to_buffer_pool[name] = buffer_pool.release();
+
+    return HAILO_SUCCESS;
+}
+
+hailo_status ServiceNetworkGroupBufferPool::reallocate_pool(const std::string &name, size_t frame_size)
+{
+    std::unique_lock<std::mutex> lock(m_mutex);
+    auto pool_size = m_output_name_to_buffer_pool[name]->buffers_count();
+    m_output_name_to_buffer_pool[name].reset();
+
+    auto buffer_pool = ServiceStreamBufferPool::create(m_vdevice_handle, frame_size,
+        pool_size, HAILO_DMA_BUFFER_DIRECTION_D2H, m_shutdown_event);
+    CHECK_EXPECTED(buffer_pool);
+    m_output_name_to_buffer_pool[name] = buffer_pool.release();
+
+    return HAILO_SUCCESS;
+}
+
+Expected<BufferPtr> ServiceNetworkGroupBufferPool::acquire_buffer(const std::string &output_name)
+{
+    CHECK_AS_EXPECTED(contains(m_output_name_to_buffer_pool, output_name), HAILO_INTERNAL_FAILURE,
+        "acquire_buffer() for output {} failed, output name does not exist in buffer pool", output_name);
+
+    std::unique_lock<std::mutex> lock(m_mutex);
+    auto buffer = m_output_name_to_buffer_pool.at(output_name)->acquire_buffer();
+    CHECK_EXPECTED(buffer);
+
+    return buffer.release();
+}
+
+hailo_status ServiceNetworkGroupBufferPool::return_to_pool(const std::string &output_name, BufferPtr buffer)
+{
+    CHECK(contains(m_output_name_to_buffer_pool, output_name), HAILO_INTERNAL_FAILURE,
+        "acquire_buffer() for output {} failed, output name does not exist in buffer pool", output_name);
+
+    std::unique_lock<std::mutex> lock(m_mutex);
+    auto status = m_output_name_to_buffer_pool.at(output_name)->return_to_pool(buffer);
+    CHECK_SUCCESS(status);
+
+    return HAILO_SUCCESS;
+}
+
+hailo_status ServiceNetworkGroupBufferPool::shutdown()
+{
+    return m_shutdown_event->signal();
+}
+
+} /* namespace hailort */

hailort/hailort_service/cng_buffer_pool.hpp

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file cng_buffer_pool.hpp
+ * @brief This model represents the buffer pools for the output reads for each network group. Used in async API
+ **/
+
+#ifndef _HAILO_CNG_BUFFER_POOL_HPP_
+#define _HAILO_CNG_BUFFER_POOL_HPP_
+
+#include "hailo/hailort.h"
+#include "hailo/hailort_common.hpp"
+#include "hailo/buffer.hpp"
+#include "hailo/vdevice.hpp"
+#include "hailo/dma_mapped_buffer.hpp"
+#include "utils/thread_safe_queue.hpp"
+
+namespace hailort
+{
+
+class ServiceStreamBufferPool
+{
+public:
+    static Expected<std::shared_ptr<ServiceStreamBufferPool>> create(uint32_t vdevice_handle, size_t buffer_size,
+        size_t buffer_count, hailo_dma_buffer_direction_t direction, EventPtr shutdown_event);
+
+    struct AllocatedMappedBuffer {
+        BufferPtr buffer;
+        DmaMappedBuffer mapped_buffer;
+    };
+
+    ServiceStreamBufferPool(size_t buffer_size, std::vector<AllocatedMappedBuffer> &&buffers,
+        SpscQueue<BufferPtr> &&m_free_buffers_queue, size_t buffers_count);
+    virtual ~ServiceStreamBufferPool() = default;
+
+    Expected<BufferPtr> acquire_buffer();
+    hailo_status return_to_pool(BufferPtr buffer);
+    size_t buffers_count();
+
+private:
+
+    size_t m_buffer_size;
+    size_t m_buffers_count;
+    std::vector<AllocatedMappedBuffer> m_buffers;
+    SpscQueue<BufferPtr> m_free_buffers_queue;
+    std::mutex m_mutex;
+};
+
+using BufferPoolPtr = std::shared_ptr<ServiceStreamBufferPool>;
+using output_name_t = std::string;
+
+// This object holds a buffer pool for each output streams of the network group.
+// It is used to pre-allocate all the buffers necessary for the reads from the device.
+// The buffers are reuseable, which also prevents allocation during inference.
+// The buffers are mapped to the device during their creation, which prevent lazy mapping each frame inference.
+// Currently only used in async API.
+class ServiceNetworkGroupBufferPool
+{
+public:
+    static Expected<std::shared_ptr<ServiceNetworkGroupBufferPool>> create(uint32_t vdevice_handle);
+
+    hailo_status allocate_pool(const std::string &name, size_t frame_size, size_t pool_size);
+    // Used in order to reallocate the pool buffers with different frame_size
+    hailo_status reallocate_pool(const std::string &name, size_t frame_size);
+
+    ServiceNetworkGroupBufferPool(ServiceNetworkGroupBufferPool &&) = delete;
+    ServiceNetworkGroupBufferPool(const ServiceNetworkGroupBufferPool &) = delete;
+    ServiceNetworkGroupBufferPool &operator=(ServiceNetworkGroupBufferPool &&) = delete;
+    ServiceNetworkGroupBufferPool &operator=(const ServiceNetworkGroupBufferPool &) = delete;
+    virtual ~ServiceNetworkGroupBufferPool() = default;
+
+    ServiceNetworkGroupBufferPool(EventPtr shutdown_event, uint32_t vdevice_handle);
+    Expected<BufferPtr> acquire_buffer(const std::string &output_name);
+    hailo_status return_to_pool(const std::string &output_name, BufferPtr buffer);
+    hailo_status shutdown();
+
+private:
+    std::unordered_map<output_name_t, BufferPoolPtr> m_output_name_to_buffer_pool;
+    EventPtr m_shutdown_event;
+    uint32_t m_vdevice_handle;
+    std::mutex m_mutex;
+};
+
+} /* namespace hailort */
+
+#endif /* _HAILO_CNG_BUFFER_POOL_HPP_ */

hailort/hailort_service/hailort_rpc_service.cpp

@@ -16,20 +16,23 @@
 #include "common/os_utils.hpp"
 
 #include "hailort_rpc_service.hpp"
+#include "cng_buffer_pool.hpp"
 #include "rpc/rpc_definitions.hpp"
 #include "service_resource_manager.hpp"
-#include "net_flow/ops/op_metadata.hpp"
+#include "net_flow/ops_metadata/op_metadata.hpp"
-#include "net_flow/ops/nms_post_process.hpp"
+#include "net_flow/ops_metadata/nms_op_metadata.hpp"
-#include "net_flow/ops/yolov8_post_process.hpp"
+#include "net_flow/ops_metadata/yolov8_op_metadata.hpp"
-#include "net_flow/ops/ssd_post_process.hpp"
+#include "net_flow/ops_metadata/ssd_op_metadata.hpp"
-#include "net_flow/ops/yolox_post_process.hpp"
+#include "net_flow/ops_metadata/yolox_op_metadata.hpp"
-#include "net_flow/ops/yolov5_op_metadata.hpp"
+#include "net_flow/ops_metadata/yolov5_op_metadata.hpp"
-#include "net_flow/ops/yolov5_seg_op_metadata.hpp"
+#include "net_flow/ops_metadata/yolov5_seg_op_metadata.hpp"
 
 #include "hef/layer_info.hpp"
 
 #include <thread>
 
+
+#define MAX_GRPC_BUFFER_SIZE (2ULL * 1024 * 1024 * 1024) // 2GB
 namespace hailort
 {
 
@@ -97,7 +100,7 @@ void HailoRtRpcService::remove_disconnected_clients()
     auto now = std::chrono::high_resolution_clock::now();
     std::set<uint32_t> pids_to_remove;
     {
-        std::unique_lock<std::mutex> lock(m_mutex);
+        std::unique_lock<std::mutex> lock(m_keep_alive_mutex);
         for (auto pid_to_last_alive : m_clients_pids) {
             auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - pid_to_last_alive.second);
             if (duration > hailort::HAILO_KEEPALIVE_INTERVAL) {
@@ -133,7 +136,7 @@ void HailoRtRpcService::keep_alive()
 
 void HailoRtRpcService::update_client_id_timestamp(uint32_t pid)
 {
-    std::unique_lock<std::mutex> lock(m_mutex);
+    std::unique_lock<std::mutex> lock(m_keep_alive_mutex);
     m_clients_pids[pid] = std::chrono::high_resolution_clock::now();
 }
 
@@ -186,7 +189,7 @@ grpc::Status HailoRtRpcService::VDevice_create(grpc::ServerContext *, const VDev
     CHECK_EXPECTED_AS_RPC_STATUS(vdevice, reply);
 
     update_client_id_timestamp(request->pid());
-    std::unique_lock<std::mutex> lock(m_vdevice_creation_mutex);
+    std::unique_lock<std::mutex> lock(m_vdevice_mutex);
     auto &vdevice_manager = ServiceResourceManager<VDevice>::get_instance();
     auto vdevice_handle = vdevice_manager.register_resource(request->pid(), std::move(vdevice.release()));
 
@@ -271,6 +274,7 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD
     }
 
     update_client_id_timestamp(request->pid());
+    std::unique_lock<std::mutex> lock(m_vdevice_mutex);
     auto lambda = [](std::shared_ptr<VDevice> vdevice, Hef &hef, NetworkGroupsParamsMap &configure_params_map) {
         return vdevice->configure(hef, configure_params_map);
     };
@@ -281,14 +285,57 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD
 
     auto &networks_manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
     for (auto network : networks.value()) {
-        auto handle = networks_manager.register_resource(request->pid(), network);
+        auto ng_handle = networks_manager.register_resource(request->pid(), network);
-        reply->add_networks_handles(handle);
+        reply->add_networks_handles(ng_handle);
+
+        bool allocate_for_raw_streams = false;
+        // The network_group's buffer pool is used for the read's buffers,
+        // On async flow - we allocate for raw-streams. This way they are already pre-allocated and mapped to the device
+        if ((configure_params_map.size() > 0) &&
+            (configure_params_map.begin()->second.stream_params_by_name.begin()->second.flags == HAILO_STREAM_FLAGS_ASYNC)) {
+            // We assume that if 1 stream is marked as ASYNC, they all are
+            allocate_for_raw_streams = true;
+        }
+        auto status = create_buffer_pools_for_ng(request->identifier().vdevice_handle(), ng_handle, request->pid(), allocate_for_raw_streams);
+        CHECK_SUCCESS_AS_RPC_STATUS(status, reply);
     }
 
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
     return grpc::Status::OK;
 }
 
+hailo_status HailoRtRpcService::create_buffer_pools_for_ng(uint32_t vdevice_handle, uint32_t ng_handle, uint32_t request_pid,
+    bool allocate_for_raw_streams)
+{
+    auto cng_buffer_pool = ServiceNetworkGroupBufferPool::create(vdevice_handle);
+    CHECK_EXPECTED_AS_STATUS(cng_buffer_pool);
+
+    auto &cng_buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
+    auto cng_buffer_pool_handle = cng_buffer_pool_manager.register_resource(request_pid, cng_buffer_pool.release());
+    CHECK(cng_buffer_pool_handle == ng_handle, HAILO_INTERNAL_FAILURE,
+        "cng_buffer_pool_handle = {} must be equal to network_group_handle ={}", cng_buffer_pool_handle, ng_handle);
+
+    if (allocate_for_raw_streams) {
+        // For Async API - The buffer size in the pool will be the stream's hw frame size as used in the infer_model pipeline
+        auto min_buffer_pool_size = get_min_buffer_pool_size(ng_handle);
+        CHECK_EXPECTED_AS_STATUS(min_buffer_pool_size);
+
+        auto streams_infos = get_all_stream_infos(ng_handle);
+        CHECK_EXPECTED_AS_STATUS(streams_infos);
+
+        for (const auto &stream_info : streams_infos.value()) {
+            if (stream_info.direction == HAILO_D2H_STREAM) {
+                auto allocate_lambda = [&](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool) {
+                    return cng_buffer_pool->allocate_pool(stream_info.name, stream_info.hw_frame_size, min_buffer_pool_size.value());
+                };
+                CHECK_SUCCESS(cng_buffer_pool_manager.execute(ng_handle, allocate_lambda));
+            }
+        }
+    }
+
+    return HAILO_SUCCESS;
+}
+
 grpc::Status HailoRtRpcService::VDevice_get_physical_devices_ids(grpc::ServerContext*,
     const VDevice_get_physical_devices_ids_Request* request, VDevice_get_physical_devices_ids_Reply* reply)
 {
@@ -370,96 +417,192 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_dup_handle(grpc::ServerCo
     return grpc::Status::OK;
 }
 
+ProtoCallbackIdentifier serialize_callback_identifier(uint32_t vdevice_handle, uint32_t ng_handle,
+    callback_type_t cb_type, const std::string &stream_name, uint32_t cb_idx,  hailo_status status, BufferPtr buffer = nullptr)
+{
+    ProtoCallbackIdentifier cb_identifier;
+    cb_identifier.set_vdevice_handle(vdevice_handle);
+    cb_identifier.set_network_group_handle(ng_handle);
+    cb_identifier.set_cb_type(cb_type);
+    cb_identifier.set_stream_name(stream_name);
+    cb_identifier.set_cb_idx(cb_idx);
+    cb_identifier.set_status(status);
+    if (buffer != nullptr) {
+        cb_identifier.set_data(buffer->data(), buffer->size());
+    }
+
+    return cb_identifier;
+}
+
 grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_release(grpc::ServerContext*, const Release_Request *request,
     Release_Reply *reply)
 {
+    auto buffer_shutdown_lambda = [](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool) {
+        return cng_buffer_pool->shutdown();
+    };
+
+    auto &buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
+    auto status = buffer_pool_manager.execute(request->network_group_identifier().network_group_handle(), buffer_shutdown_lambda);
+    CHECK_SUCCESS_AS_RPC_STATUS(status, reply);
+    buffer_pool_manager.release_resource(request->network_group_identifier().network_group_handle(), request->pid());
+
     auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
     manager.release_resource(request->network_group_identifier().network_group_handle(), request->pid());
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
     return grpc::Status::OK;
 }
 
-grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_infer_async(grpc::ServerContext*,
+hailo_status HailoRtRpcService::add_input_named_buffer(const ProtoTransferRequest &proto_stream_transfer_request,
-    const ConfiguredNetworkGroup_infer_async_Request *request, ConfiguredNetworkGroup_infer_async_Reply *reply)
+    uint32_t vdevice_handle, uint32_t ng_handle, std::shared_ptr<ConfiguredNetworkGroup_infer_async_Request> infer_async_request,
+    NamedBuffersCallbacks &named_buffers_callbacks)
 {
+    // Prepare input buffer
+    BufferPtr buffer;
+    MemoryView mem_view;
+    auto *data = reinterpret_cast<const uint8_t*>(proto_stream_transfer_request.data().c_str());
+    if (reinterpret_cast<size_t>(data) % HailoRTCommon::HW_DATA_ALIGNMENT == 0) {
+        // Input buffers is aligned to 8
+        mem_view = MemoryView::create_const(data, proto_stream_transfer_request.data().size());
+    } else {
+        // The memory is not aligned to 8, therefore we need to copy the data into a buffer
+        auto buffer_exp = Buffer::create_shared(data, proto_stream_transfer_request.data().size(),
+            BufferStorageParams::create_dma());
+        CHECK_EXPECTED(buffer_exp);
+        buffer = buffer_exp.release();
+        mem_view = MemoryView(*buffer);
+    }
+
+    // Preparing callback
+    auto &stream_name = proto_stream_transfer_request.stream_name();
+    auto cb_idx = proto_stream_transfer_request.cb_idx();
+    std::function<void(hailo_status)> transfer_done = [this, vdevice_handle, ng_handle, cb_idx, stream_name, buffer, infer_async_request]
+        (hailo_status status)
+    {
+        // We pass the request (which is shared_ptr) to the callback in order to keep the input's memory alive until inference is done.
+        (void)infer_async_request;
+        (void)buffer;
+
+        auto cb_identifier = serialize_callback_identifier(vdevice_handle, ng_handle, CALLBACK_TYPE_TRANSFER,
+            stream_name, cb_idx, status);
+        enqueue_cb_identifier(vdevice_handle, std::move(cb_identifier));
+    };
+
+    named_buffers_callbacks.emplace(stream_name, std::make_pair(mem_view, transfer_done));
+    return HAILO_SUCCESS;
+}
+
+hailo_status HailoRtRpcService::add_output_named_buffer(const ProtoTransferRequest &proto_stream_transfer_request, uint32_t vdevice_handle,
+    uint32_t ng_handle, NamedBuffersCallbacks &named_buffers_callbacks)
+{
+    // Prepare output buffer
+    auto &stream_name = proto_stream_transfer_request.stream_name();
+    auto buffer_exp = acquire_buffer_from_cng_pool(ng_handle, stream_name);
+    CHECK_EXPECTED(buffer_exp);
+    auto buffer = buffer_exp.release();
+
+    // Prepare callback
+    auto cb_idx = proto_stream_transfer_request.cb_idx();
+    std::function<void(hailo_status)> transfer_done = [this, vdevice_handle, ng_handle, cb_idx, stream_name, buffer]
+        (hailo_status status)
+    {
+        auto cb_identifier = serialize_callback_identifier(vdevice_handle, ng_handle, CALLBACK_TYPE_TRANSFER,
+            stream_name, cb_idx, status, buffer);
+        return_buffer_to_cng_pool(ng_handle, stream_name, buffer);
+        enqueue_cb_identifier(vdevice_handle, std::move(cb_identifier));
+    };
+
+    named_buffers_callbacks.emplace(stream_name, std::make_pair(MemoryView(*buffer), transfer_done));
+    return HAILO_SUCCESS;
+}
+
+Expected<NamedBuffersCallbacks> HailoRtRpcService::prepare_named_buffers_callbacks(uint32_t vdevice_handle,
+    uint32_t ng_handle, std::shared_ptr<ConfiguredNetworkGroup_infer_async_Request> infer_async_request)
+{
+    NamedBuffersCallbacks named_buffers_callbacks;
+    for (const auto &proto_stream_transfer_request : infer_async_request->transfer_requests()) {
+        auto direction = proto_stream_transfer_request.direction();
+        auto status = HAILO_SUCCESS;
+        if (direction == HAILO_H2D_STREAM) {
+            status = add_input_named_buffer(proto_stream_transfer_request, vdevice_handle, ng_handle, infer_async_request, named_buffers_callbacks);
+        } else {
+            status = add_output_named_buffer(proto_stream_transfer_request, vdevice_handle, ng_handle, named_buffers_callbacks);
+        }
+        CHECK_SUCCESS_AS_EXPECTED(status);
+    }
+
+    return named_buffers_callbacks;
+}
+
+void HailoRtRpcService::enqueue_cb_identifier(uint32_t vdevice_handle, ProtoCallbackIdentifier &&cb_identifier)
+{
+    auto lambda = [](std::shared_ptr<VDeviceCallbacksQueue> cb_queue, ProtoCallbackIdentifier &cb_identifier) {
+        return cb_queue->enqueue(std::move(cb_identifier));
+    };
+
+    auto &cb_queue_manager = ServiceResourceManager<VDeviceCallbacksQueue>::get_instance();
+    auto status = cb_queue_manager.execute(vdevice_handle, lambda, std::move(cb_identifier));
+    if (status != HAILO_SUCCESS) {
+        LOGGER__ERROR("Failed to enqueue callback to VDeviceCallbacksQueue with status={}", status);
+    }
+}
+
+hailo_status HailoRtRpcService::return_buffer_to_cng_pool(uint32_t ng_handle, const std::string &output_name, BufferPtr buffer)
+{
+    auto &cng_buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
+    auto lambda_return_to_pool = [](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool,
+        const std::string &stream_name, BufferPtr buffer) {
+        return cng_buffer_pool->return_to_pool(stream_name, buffer);
+    };
+    auto status = cng_buffer_pool_manager.execute(ng_handle, lambda_return_to_pool,
+        output_name, buffer);
+    CHECK_SUCCESS(status);
+
+    return HAILO_SUCCESS;
+}
+
+Expected<BufferPtr> HailoRtRpcService::acquire_buffer_from_cng_pool(uint32_t ng_handle, const std::string &output_name)
+{
+    auto &cng_buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
+    auto lambda_acquire_buffer = [](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool, const std::string &output_name) {
+        return cng_buffer_pool->acquire_buffer(output_name);
+    };
+    auto buffer = cng_buffer_pool_manager.execute<Expected<BufferPtr>>(ng_handle, lambda_acquire_buffer, output_name);
+    CHECK_EXPECTED(buffer);
+    return buffer.release();
+}
+
+grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_infer_async(grpc::ServerContext*,
+    const ConfiguredNetworkGroup_infer_async_Request *raw_request, ConfiguredNetworkGroup_infer_async_Reply *reply)
+{
+    // Moving ownership of the request, so we can use the request's memory as the input buffers instead of allocating new memory for it.
+    auto request = make_shared_nothrow<ConfiguredNetworkGroup_infer_async_Request>(std::move(*raw_request));
+
     auto vdevice_handle = request->identifier().vdevice_handle();
     auto ng_handle = request->identifier().network_group_handle();
     auto infer_request_done_cb_idx = request->infer_request_done_cb_idx();
 
-    NamedBuffersCallbacks named_buffers_callbacks;
+    // Prepare buffers
-    for (const auto &proto_transfer_request : request->transfer_requests()) {
+    auto named_buffers_callbacks = prepare_named_buffers_callbacks(vdevice_handle, ng_handle, request);
-        auto &stream_name = proto_transfer_request.stream_name();
+    CHECK_EXPECTED_AS_RPC_STATUS(named_buffers_callbacks, reply);
-        auto direction = proto_transfer_request.direction();
-        auto cb_idx = proto_transfer_request.cb_idx();
-        BufferPtr buffer;
-        if (direction == HAILO_H2D_STREAM) {
-            // TODO: Remove memcpy after HRT-12238
-            auto buffer_exp = Buffer::create_shared(reinterpret_cast<const uint8_t*>(proto_transfer_request.data().c_str()),
-                proto_transfer_request.size(), BufferStorageParams::create_dma());
-            CHECK_EXPECTED_AS_RPC_STATUS(buffer_exp, reply);
-            buffer = buffer_exp.release();
-        } else {
-            // TODO: HRT-12360 - Use buffer pool for the service reads
-            auto buffer_exp = Buffer::create_shared(proto_transfer_request.size(), BufferStorageParams::create_dma());
-            CHECK_EXPECTED_AS_RPC_STATUS(buffer_exp, reply);
-            buffer = buffer_exp.release();
-        }
 
-        std::function<void(hailo_status)> transfer_done = [vdevice_handle, ng_handle, cb_idx, stream_name, direction, buffer]
+    // Prepare request finish callback
-            (hailo_status status)
+    auto infer_request_done_cb = [this, vdevice_handle, ng_handle, infer_request_done_cb_idx](hailo_status status) {
-        {
+        auto cb_identifier = serialize_callback_identifier(vdevice_handle, ng_handle, CALLBACK_TYPE_INFER_REQUEST,
-            ProtoCallbackIdentifier cb_identifier;
+            "", infer_request_done_cb_idx, status);
-            cb_identifier.set_vdevice_handle(vdevice_handle);
+        enqueue_cb_identifier(vdevice_handle, std::move(cb_identifier));
-            cb_identifier.set_network_group_handle(ng_handle);
-            cb_identifier.set_cb_type(CALLBACK_TYPE_TRANSFER);
-            cb_identifier.set_stream_name(stream_name);
-            cb_identifier.set_cb_idx(cb_idx);
-            cb_identifier.set_status(status);
-
-            auto lambda = [direction](std::shared_ptr<VDeviceCallbacksQueue> cb_queue, ProtoCallbackIdentifier &cb_identifier, BufferPtr buffer) {
-                if (direction == HAILO_D2H_STREAM) {
-                    cb_identifier.set_data(buffer->data(), buffer->size());
-                }
-                return cb_queue->enqueue(std::move(cb_identifier));
-            };
-
-            auto &cb_queue_manager = ServiceResourceManager<VDeviceCallbacksQueue>::get_instance();
-            auto exc_status = cb_queue_manager.execute(vdevice_handle, lambda, std::move(cb_identifier), buffer);
-            if (exc_status != HAILO_SUCCESS) {
-                LOGGER__ERROR("Failed to enqueue callback to VDeviceCallbacksQueue with status={}", status);
-            }
-        };
-        named_buffers_callbacks.emplace(stream_name, std::make_pair(MemoryView(*buffer), transfer_done));
-    }
-
-    auto infer_request_done_cb = [vdevice_handle, ng_handle, infer_request_done_cb_idx](hailo_status status){
-        ProtoCallbackIdentifier cb_identifier;
-        cb_identifier.set_vdevice_handle(vdevice_handle);
-        cb_identifier.set_network_group_handle(ng_handle);
-        cb_identifier.set_cb_type(CALLBACK_TYPE_INFER_REQUEST);
-        cb_identifier.set_cb_idx(infer_request_done_cb_idx);
-
-        auto lambda = [](std::shared_ptr<VDeviceCallbacksQueue> cb_queue, ProtoCallbackIdentifier &cb_identifier) {
-            return cb_queue->enqueue(std::move(cb_identifier));
-        };
-
-        auto &cb_queue_manager = ServiceResourceManager<VDeviceCallbacksQueue>::get_instance();
-        auto exc_status = cb_queue_manager.execute(vdevice_handle, lambda, std::move(cb_identifier));
-        if (exc_status != HAILO_SUCCESS) {
-            LOGGER__ERROR("Failed to enqueue callback to VDeviceCallbacksQueue with status={}", status);
-        }
     };
 
+    // Run infer async
     auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng, NamedBuffersCallbacks &named_buffers_callbacks,
         const std::function<void(hailo_status)> &infer_request_done_cb) {
             return cng->infer_async(named_buffers_callbacks, infer_request_done_cb);
     };
 
     auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
-    auto status = manager.execute(request->identifier().network_group_handle(), lambda, named_buffers_callbacks, infer_request_done_cb);
+    auto status = manager.execute(request->identifier().network_group_handle(), lambda, named_buffers_callbacks.release(), infer_request_done_cb);
-    if (HAILO_STREAM_ABORTED_BY_USER == status) {
+    if (HAILO_STREAM_ABORT == status) {
         LOGGER__INFO("User aborted inference");
-        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORTED_BY_USER));
+        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORT));
         return grpc::Status::OK;
     }
     CHECK_SUCCESS_AS_RPC_STATUS(status, reply);
@@ -629,7 +772,7 @@ void serialize_vstream_info(const hailo_vstream_info_t &info, ProtoVStreamInfo *
         auto nms_shape_proto = info_proto->mutable_nms_shape();
         nms_shape_proto->set_number_of_classes(info.nms_shape.number_of_classes);
         nms_shape_proto->set_max_bbox_per_class(info.nms_shape.max_bboxes_per_class);
-        nms_shape_proto->set_max_mask_size(info.nms_shape.max_mask_size);
+        nms_shape_proto->set_max_accumulated_mask_size(info.nms_shape.max_accumulated_mask_size);
     } else {
         auto shape_proto = info_proto->mutable_shape();
         shape_proto->set_height(info.shape.height);
@@ -918,6 +1061,7 @@ void serialize_yolov5seg_op_metadata(hailort::net_flow::OpMetadata &op_metadata,
 
     yolov5seg_config_proto->set_mask_threshold(yolov5seg_config.mask_threshold);
     yolov5seg_config_proto->set_layer_name(yolov5seg_config.proto_layer_name);
+    yolov5seg_config_proto->set_max_accumulated_mask_size(yolov5seg_config.max_accumulated_mask_size);
 }
 
 void serialize_op_matadata(hailort::net_flow::OpMetadata &op_metadata, ProtoOpMetadata *op_metadata_proto)
@@ -1026,12 +1170,7 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_all_vstream_infos(grp
     const ConfiguredNetworkGroup_get_vstream_infos_Request *request,
     ConfiguredNetworkGroup_get_vstream_infos_Reply *reply)
 {
-    auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng, std::string network_name) {
+    auto expected_vstream_infos = get_all_vstream_infos(request->identifier().network_group_handle());
-        return cng->get_all_vstream_infos(network_name);
-    };
-    auto &net_group_manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
-    auto expected_vstream_infos = net_group_manager.execute<Expected<std::vector<hailo_vstream_info_t>>>(
-        request->identifier().network_group_handle(), lambda, request->network_name());
     CHECK_EXPECTED_AS_RPC_STATUS(expected_vstream_infos, reply);
 
     serialize_vstream_infos(reply, expected_vstream_infos.value());
@@ -1170,7 +1309,6 @@ grpc::Status HailoRtRpcService::InputVStreams_create(grpc::ServerContext *, cons
     auto &net_group_manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
     net_group_manager.dup_handle(network_group_handle, client_pid);
 
-
     auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng, const std::map<std::string, hailo_vstream_params_t> &inputs_params) {
         return cng->create_input_vstreams(inputs_params);
     };
@@ -1178,13 +1316,12 @@ grpc::Status HailoRtRpcService::InputVStreams_create(grpc::ServerContext *, cons
     CHECK_EXPECTED_AS_RPC_STATUS(vstreams_expected, reply);
     auto vstreams = vstreams_expected.release();
 
-    auto &manager = ServiceResourceManager<InputVStream>::get_instance();
+    auto &vstreams_manager = ServiceResourceManager<InputVStream>::get_instance();
     for (size_t i = 0; i < vstreams.size(); i++) {
-        auto handle = manager.register_resource(client_pid, make_shared_nothrow<InputVStream>(std::move(vstreams[i])));
+        auto handle = vstreams_manager.register_resource(client_pid, make_shared_nothrow<InputVStream>(std::move(vstreams[i])));
         reply->add_handles(handle);
     }
 
-
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
     return grpc::Status::OK;
 }
@@ -1237,13 +1374,19 @@ grpc::Status HailoRtRpcService::OutputVStreams_create(grpc::ServerContext *, con
     CHECK_EXPECTED_AS_RPC_STATUS(vstreams_expected, reply);
     auto vstreams = vstreams_expected.release();
 
-    auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
+    // The network_group's buffer pool is used for the read's buffers.
+    auto &cng_buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
+    auto &vstream_manager = ServiceResourceManager<OutputVStream>::get_instance();
     for (size_t i = 0; i < vstreams.size(); i++) {
-        auto handle = manager.register_resource(client_pid, make_shared_nothrow<OutputVStream>(std::move(vstreams[i])));
+        auto allocate_lambda = [&](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool) {
+            return cng_buffer_pool->allocate_pool(vstreams[i].name(), vstreams[i].get_frame_size(), output_params.at(vstreams[i].name()).queue_size);
+        };
+        CHECK_SUCCESS_AS_RPC_STATUS(cng_buffer_pool_manager.execute(network_group_handle, allocate_lambda), reply);
+
+        auto handle = vstream_manager.register_resource(client_pid, make_shared_nothrow<OutputVStream>(std::move(vstreams[i])));
         reply->add_handles(handle);
     }
 
-
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
     return grpc::Status::OK;
 }
@@ -1292,16 +1435,17 @@ grpc::Status HailoRtRpcService::InputVStream_is_multi_planar(grpc::ServerContext
 grpc::Status HailoRtRpcService::InputVStream_write(grpc::ServerContext*, const InputVStream_write_Request *request,
         InputVStream_write_Reply *reply)
 {
-    std::vector<uint8_t> data(request->data().begin(), request->data().end());
+    MemoryView mem_view = MemoryView::create_const(reinterpret_cast<const uint8_t*>(request->data().c_str()),
+        request->data().size());
     auto lambda = [](std::shared_ptr<InputVStream> input_vstream, const MemoryView &buffer) {
         return input_vstream->write(std::move(buffer));
     };
     auto &manager = ServiceResourceManager<InputVStream>::get_instance();
-    auto status = manager.execute(request->identifier().vstream_handle(), lambda, MemoryView::create_const(data.data(), data.size()));
+    auto status = manager.execute(request->identifier().vstream_handle(), lambda, mem_view);
 
-    if (HAILO_STREAM_ABORTED_BY_USER == status) {
+    if (HAILO_STREAM_ABORT == status) {
         LOGGER__INFO("User aborted VStream write.");
-        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORTED_BY_USER));
+        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORT));
         return grpc::Status::OK;
     }
     CHECK_SUCCESS_AS_RPC_STATUS(status,  reply, "VStream write failed");
@@ -1315,6 +1459,7 @@ grpc::Status HailoRtRpcService::InputVStream_write_pix(grpc::ServerContext*, con
     hailo_pix_buffer_t pix_buffer = {};
     pix_buffer.index = request->index();
     pix_buffer.number_of_planes = request->number_of_planes();
+    pix_buffer.memory_type = HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR;  // Service does not support other memory types
     std::vector<std::vector<uint8_t>> data_arrays;
     data_arrays.reserve(pix_buffer.number_of_planes);
     for (uint32_t i =0; i < pix_buffer.number_of_planes; i++) {
@@ -1329,9 +1474,9 @@ grpc::Status HailoRtRpcService::InputVStream_write_pix(grpc::ServerContext*, con
     auto &manager = ServiceResourceManager<InputVStream>::get_instance();
     auto status = manager.execute(request->identifier().vstream_handle(), lambda, pix_buffer);
 
-    if (HAILO_STREAM_ABORTED_BY_USER == status) {
+    if (HAILO_STREAM_ABORT == status) {
         LOGGER__INFO("User aborted VStream write.");
-        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORTED_BY_USER));
+        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORT));
         return grpc::Status::OK;
     }
     CHECK_SUCCESS_AS_RPC_STATUS(status,  reply, "VStream write failed");
@@ -1362,34 +1507,71 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_network_infos(grpc::S
 grpc::Status HailoRtRpcService::OutputVStream_read(grpc::ServerContext*, const OutputVStream_read_Request *request,
     OutputVStream_read_Reply *reply)
 {
-    std::vector<uint8_t> data(request->size());
+    auto ng_handle = request->identifier().network_group_handle();
+    auto vstream_name = output_vstream_name(request->identifier().vstream_handle());
+    CHECK_EXPECTED_AS_RPC_STATUS(vstream_name, reply);
+
+    auto buffer_exp = acquire_buffer_from_cng_pool(ng_handle, vstream_name.value());
+    CHECK_EXPECTED_AS_RPC_STATUS(buffer_exp, reply);
+    auto buffer = buffer_exp.release();
+
     auto lambda = [](std::shared_ptr<OutputVStream> output_vstream, MemoryView &buffer) {
         return output_vstream->read(std::move(buffer));
     };
     auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
-    auto status = manager.execute(request->identifier().vstream_handle(), lambda, MemoryView(data.data(), data.size()));
+    auto status = manager.execute(request->identifier().vstream_handle(), lambda, MemoryView(buffer->data(), buffer->size()));
 
-    if (HAILO_STREAM_ABORTED_BY_USER == status) {
+    if (HAILO_STREAM_ABORT == status) {
         LOGGER__INFO("User aborted VStream read.");
-        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORTED_BY_USER));
+        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORT));
         return grpc::Status::OK;
     }
     CHECK_SUCCESS_AS_RPC_STATUS(status,  reply, "VStream read failed");
-    reply->set_data(data.data(), data.size());
+
+    if (buffer->size() > MAX_GRPC_BUFFER_SIZE) {
+        LOGGER__ERROR("Response buffer size is too big: {}. Max response size is: {}", buffer->size(), MAX_GRPC_BUFFER_SIZE);
+        reply->set_status(static_cast<uint32_t>(HAILO_RPC_FAILED));
+        return grpc::Status::OK;
+    }
+
+    reply->set_data(buffer->data(), buffer->size());
+
+    status = return_buffer_to_cng_pool(ng_handle, vstream_name.value(), buffer);
+    CHECK_SUCCESS_AS_RPC_STATUS(status, reply);
+
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
     return grpc::Status::OK;
 }
 
+Expected<std::vector<hailo_stream_info_t>> HailoRtRpcService::get_all_stream_infos(uint32_t ng_handle)
+{
+    auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng) {
+        return cng->get_all_stream_infos();
+    };
+    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto expected_stream_infos = manager.execute<Expected<std::vector<hailo_stream_info_t>>>(ng_handle, lambda);
+    CHECK_EXPECTED(expected_stream_infos);
+
+    return expected_stream_infos.release();
+}
+
+Expected<std::vector<hailo_vstream_info_t>> HailoRtRpcService::get_all_vstream_infos(uint32_t ng_handle)
+{
+    auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng) {
+        return cng->get_all_vstream_infos();
+    };
+    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto expected_vstream_infos = manager.execute<Expected<std::vector<hailo_vstream_info_t>>>(ng_handle, lambda);
+    CHECK_EXPECTED(expected_vstream_infos);
+
+    return expected_vstream_infos.release();
+}
+
 grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_all_stream_infos(grpc::ServerContext*,
     const ConfiguredNetworkGroup_get_all_stream_infos_Request *request,
     ConfiguredNetworkGroup_get_all_stream_infos_Reply *reply)
 {
-    auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng) {
+    auto expected_stream_infos = get_all_stream_infos(request->identifier().network_group_handle());
-        return cng->get_all_stream_infos();
-    };
-    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
-    auto expected_stream_infos = manager.execute<Expected<std::vector<hailo_stream_info_t>>>(request->identifier().network_group_handle(),
-        lambda);
     CHECK_EXPECTED_AS_RPC_STATUS(expected_stream_infos, reply);
 
     auto proto_stream_infos = reply->mutable_stream_infos();
@@ -1495,16 +1677,23 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_sorted_output_names(g
     return grpc::Status::OK;
 }
 
-grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_min_buffer_pool_size(grpc::ServerContext*,
+Expected<size_t> HailoRtRpcService::get_min_buffer_pool_size(uint32_t ng_handle)
-    const ConfiguredNetworkGroup_get_min_buffer_pool_size_Request *request,
-    ConfiguredNetworkGroup_get_min_buffer_pool_size_Reply *reply)
 {
     auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng) {
         return cng->get_min_buffer_pool_size();
     };
     auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
-    auto min_buffer_pool_size_expected = manager.execute<Expected<size_t>>(request->identifier().network_group_handle(),
+    auto min_buffer_pool_size = manager.execute<Expected<size_t>>(ng_handle, lambda);
-        lambda);
+    CHECK_EXPECTED(min_buffer_pool_size);
+
+    return min_buffer_pool_size.release();
+}
+
+grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_min_buffer_pool_size(grpc::ServerContext*,
+    const ConfiguredNetworkGroup_get_min_buffer_pool_size_Request *request,
+    ConfiguredNetworkGroup_get_min_buffer_pool_size_Reply *reply)
+{
+    auto min_buffer_pool_size_expected = get_min_buffer_pool_size(request->identifier().network_group_handle());
     CHECK_EXPECTED_AS_RPC_STATUS(min_buffer_pool_size_expected, reply);
 
     reply->set_min_buffer_pool_size(static_cast<uint32_t>(min_buffer_pool_size_expected.release()));
@@ -1598,6 +1787,22 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_nms_max_bboxes_per_cl
     return grpc::Status::OK;
 }
 
+grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size(grpc::ServerContext*,
+    const ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size_Request *request,
+    ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size_Reply *reply)
+{
+    auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng, const std::string &edge_name, uint32_t max_accumulated_mask_size) {
+        return cng->set_nms_max_accumulated_mask_size(edge_name, max_accumulated_mask_size);
+    };
+    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto status = manager.execute(request->identifier().network_group_handle(), lambda,
+        request->edge_name(), request->max_accumulated_mask_size());
+    CHECK_SUCCESS_AS_RPC_STATUS(status, reply);
+
+    reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
+    return grpc::Status::OK;
+}
+
 grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_stream_names_from_vstream_name(grpc::ServerContext*,
     const ConfiguredNetworkGroup_get_stream_names_from_vstream_name_Request *request,
     ConfiguredNetworkGroup_get_stream_names_from_vstream_name_Reply *reply)
@@ -1656,11 +1861,7 @@ grpc::Status HailoRtRpcService::InputVStream_get_frame_size(grpc::ServerContext*
 grpc::Status HailoRtRpcService::OutputVStream_get_frame_size(grpc::ServerContext*, const VStream_get_frame_size_Request *request,
     VStream_get_frame_size_Reply *reply)
 {
-    auto lambda = [](std::shared_ptr<OutputVStream> output_vstream) {
+    auto frame_size = output_vstream_frame_size(request->identifier().vstream_handle());
-        return output_vstream->get_frame_size();
-    };
-    auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
-    auto frame_size = manager.execute<Expected<size_t>>(request->identifier().vstream_handle(), lambda);
     CHECK_EXPECTED_AS_RPC_STATUS(frame_size, reply);
 
     reply->set_frame_size(static_cast<uint32_t>(frame_size.release()));
@@ -1692,14 +1893,34 @@ grpc::Status HailoRtRpcService::InputVStream_name(grpc::ServerContext*, const VS
     return grpc::Status::OK;
 }
 
-grpc::Status HailoRtRpcService::OutputVStream_name(grpc::ServerContext*, const VStream_name_Request *request,
+Expected<std::string> HailoRtRpcService::output_vstream_name(uint32_t vstream_handle)
-    VStream_name_Reply *reply)
 {
     auto lambda = [](std::shared_ptr<OutputVStream> output_vstream) {
         return output_vstream->name();
     };
     auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
-    auto name = manager.execute<Expected<std::string>>(request->identifier().vstream_handle(), lambda);
+    auto name = manager.execute<Expected<std::string>>(vstream_handle, lambda);
+    CHECK_EXPECTED(name);
+
+    return name.release();
+}
+
+Expected<size_t> HailoRtRpcService::output_vstream_frame_size(uint32_t vstream_handle)
+{
+    auto lambda = [](std::shared_ptr<OutputVStream> output_vstream) {
+        return output_vstream->get_frame_size();
+    };
+    auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
+    auto frame_size = manager.execute<Expected<size_t>>(vstream_handle, lambda);
+    CHECK_EXPECTED(frame_size);
+
+    return frame_size.release();
+}
+
+grpc::Status HailoRtRpcService::OutputVStream_name(grpc::ServerContext*, const VStream_name_Request *request,
+    VStream_name_Reply *reply)
+{
+    auto name = output_vstream_name(request->identifier().vstream_handle());
     CHECK_EXPECTED_AS_RPC_STATUS(name, reply);
 
     reply->set_name(name.release());
@@ -1971,6 +2192,23 @@ grpc::Status HailoRtRpcService::OutputVStream_set_nms_iou_threshold(grpc::Server
     return grpc::Status::OK;
 }
 
+hailo_status HailoRtRpcService::update_buffer_size_in_pool(uint32_t vstream_handle, uint32_t network_group_handle)
+{
+    auto vstream_name = output_vstream_name(vstream_handle);
+    CHECK_EXPECTED(vstream_name);
+
+    auto frame_size = output_vstream_frame_size(vstream_handle);
+    CHECK_EXPECTED(frame_size);
+
+    auto &cng_buffer_pool_manager = ServiceResourceManager<ServiceNetworkGroupBufferPool>::get_instance();
+    auto allocate_lambda = [&](std::shared_ptr<ServiceNetworkGroupBufferPool> cng_buffer_pool) {
+        return cng_buffer_pool->reallocate_pool(vstream_name.release(), frame_size.release());
+    };
+    CHECK_SUCCESS(cng_buffer_pool_manager.execute(network_group_handle, allocate_lambda));
+
+    return HAILO_SUCCESS;
+}
+
 grpc::Status HailoRtRpcService::OutputVStream_set_nms_max_proposals_per_class(grpc::ServerContext*, const VStream_set_nms_max_proposals_per_class_Request *request,
     VStream_set_nms_max_proposals_per_class_Reply *reply)
 {
@@ -1981,6 +2219,26 @@ grpc::Status HailoRtRpcService::OutputVStream_set_nms_max_proposals_per_class(gr
     auto status = manager.execute(request->identifier().vstream_handle(), lambda, static_cast<uint32_t>(request->max_proposals_per_class()));
     CHECK_SUCCESS_AS_RPC_STATUS(status,  reply, "set_nms_max_proposals_per_class failed");
 
+    status = update_buffer_size_in_pool(request->identifier().vstream_handle(), request->identifier().network_group_handle());
+    CHECK_SUCCESS_AS_RPC_STATUS(status, reply, "Updating buffer size in pool failed");
+
+    reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
+    return grpc::Status::OK;
+}
+
+grpc::Status HailoRtRpcService::OutputVStream_set_nms_max_accumulated_mask_size(grpc::ServerContext*,
+    const VStream_set_nms_max_accumulated_mask_size_Request *request, VStream_set_nms_max_accumulated_mask_size_Reply *reply)
+{
+    auto lambda = [](std::shared_ptr<OutputVStream> output_vstream, uint32_t max_accumulated_mask_size) {
+        return output_vstream->set_nms_max_accumulated_mask_size(max_accumulated_mask_size);
+    };
+    auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
+    auto status = manager.execute(request->identifier().vstream_handle(), lambda, static_cast<uint32_t>(request->max_accumulated_mask_size()));
+    CHECK_SUCCESS_AS_RPC_STATUS(status,  reply, "set_nms_max_accumulated_mask_size failed");
+
+    status = update_buffer_size_in_pool(request->identifier().vstream_handle(), request->identifier().network_group_handle());
+    CHECK_SUCCESS_AS_RPC_STATUS(status, reply, "Updating buffer size in pool failed");
+
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
     return grpc::Status::OK;
 }

hailort/hailort_service/hailort_rpc_service.hpp

@@ -123,6 +123,8 @@ public:
         const VStream_set_nms_iou_threshold_Request *request, VStream_set_nms_iou_threshold_Reply*) override;
     virtual grpc::Status OutputVStream_set_nms_max_proposals_per_class(grpc::ServerContext *ctx,
         const VStream_set_nms_max_proposals_per_class_Request *request, VStream_set_nms_max_proposals_per_class_Reply*) override;
+    virtual grpc::Status OutputVStream_set_nms_max_accumulated_mask_size(grpc::ServerContext *ctx,
+        const VStream_set_nms_max_accumulated_mask_size_Request *request, VStream_set_nms_max_accumulated_mask_size_Reply*) override;
 
     virtual grpc::Status ConfiguredNetworkGroup_dup_handle(grpc::ServerContext *ctx, const ConfiguredNetworkGroup_dup_handle_Request *request,
         ConfiguredNetworkGroup_dup_handle_Reply*) override;
@@ -206,6 +208,9 @@ public:
     virtual grpc::Status ConfiguredNetworkGroup_set_nms_max_bboxes_per_class(grpc::ServerContext*,
         const ConfiguredNetworkGroup_set_nms_max_bboxes_per_class_Request *request,
         ConfiguredNetworkGroup_set_nms_max_bboxes_per_class_Reply *reply) override;
+    virtual grpc::Status ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size(grpc::ServerContext*,
+        const ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size_Request *request,
+        ConfiguredNetworkGroup_set_nms_max_accumulated_mask_size_Reply *reply) override;
     virtual grpc::Status ConfiguredNetworkGroup_get_stream_names_from_vstream_name(grpc::ServerContext*,
         const ConfiguredNetworkGroup_get_stream_names_from_vstream_name_Request *request,
         ConfiguredNetworkGroup_get_stream_names_from_vstream_name_Reply *reply) override;
@@ -224,12 +229,30 @@ private:
     void abort_vstreams_by_pids(std::set<uint32_t> &pids);
     void remove_disconnected_clients();
     void update_client_id_timestamp(uint32_t pid);
+    Expected<size_t> get_min_buffer_pool_size(uint32_t ng_handle);
+    Expected<std::vector<hailo_stream_info_t>> get_all_stream_infos(uint32_t ng_handle);
+    Expected<std::vector<hailo_vstream_info_t>> get_all_vstream_infos(uint32_t ng_handle);
+    Expected<std::string> output_vstream_name(uint32_t vstream_handle);
+    hailo_status create_buffer_pools_for_ng(uint32_t vdevice_handle, uint32_t ng_handle, uint32_t request_pid,
+        bool allocate_for_raw_streams);
+    Expected<NamedBuffersCallbacks> prepare_named_buffers_callbacks(uint32_t vdevice_handle,
+        uint32_t ng_handle, std::shared_ptr<ConfiguredNetworkGroup_infer_async_Request> infer_async_request);
+    hailo_status add_input_named_buffer(const ProtoTransferRequest &proto_stream_transfer_request, uint32_t vdevice_handle,
+        uint32_t ng_handle, std::shared_ptr<ConfiguredNetworkGroup_infer_async_Request> infer_async_request,
+        NamedBuffersCallbacks &named_buffers_callbacks);
+    hailo_status add_output_named_buffer(const ProtoTransferRequest &proto_stream_transfer_request, uint32_t vdevice_handle,
+        uint32_t ng_handle, NamedBuffersCallbacks &named_buffers_callbacks);
+    void enqueue_cb_identifier(uint32_t vdevice_handle, ProtoCallbackIdentifier &&cb_identifier);
+    hailo_status return_buffer_to_cng_pool(uint32_t ng_handle, const std::string &output_name, BufferPtr buffer);
+    Expected<BufferPtr> acquire_buffer_from_cng_pool(uint32_t ng_handle, const std::string &output_name);
+    Expected<size_t> output_vstream_frame_size(uint32_t vstream_handle);
+    hailo_status update_buffer_size_in_pool(uint32_t vstream_handle, uint32_t network_group_handle);
 
-    std::mutex m_mutex;
+    std::mutex m_keep_alive_mutex;
     std::map<uint32_t, std::chrono::time_point<std::chrono::high_resolution_clock>> m_clients_pids;
     std::unique_ptr<std::thread> m_keep_alive;
 
-    std::mutex m_vdevice_creation_mutex;
+    std::mutex m_vdevice_mutex;
 };
 
 }

hailort/hailortcli/CMakeLists.txt

@@ -72,6 +72,7 @@ target_link_libraries(hailortcli
     nlohmann_json
     spdlog::spdlog
     readerwriterqueue
+    eigen
     DotWriter
     scheduler_mon_proto
     profiler_proto)

hailort/hailortcli/benchmark_command.cpp

@@ -98,10 +98,10 @@ hailo_status BenchmarkCommand::execute()
         std::cout << "FPS     (hw_only)                 = " << hw_only_res.fps().value() <<std::endl;
         std::cout << "        (streaming)               = " << streaming_res->fps().value() <<std::endl;
         if (auto hw_latency = latency_res->hw_latency()) {
-            std::cout << "Latency (hw)                      = " << InferResultsFormatUtils::latency_result_to_ms(hw_latency.value()) << " ms" << std::endl;
+            std::cout << "Latency (hw)                      = " << InferStatsPrinter::latency_result_to_ms(hw_latency.value()) << " ms" << std::endl;
         }
         if (auto overall_latency = latency_res->overall_latency()) {
-            std::cout << "        (overall)                 = " << InferResultsFormatUtils::latency_result_to_ms(overall_latency.value()) << " ms" << std::endl;
+            std::cout << "        (overall)                 = " << InferStatsPrinter::latency_result_to_ms(overall_latency.value()) << " ms" << std::endl;
         }
     }
     if (!m_not_measure_power) {

hailort/hailortcli/download_action_list_command.cpp

@@ -411,7 +411,7 @@ Expected<ordered_json> DownloadActionListCommand::parse_single_action(uint32_t b
 }
 
 Expected<ordered_json> DownloadActionListCommand::parse_context(Device &device, uint32_t network_group_id,
-    CONTROL_PROTOCOL__context_switch_context_type_t context_type, uint8_t context_index, const std::string &context_name)
+    CONTROL_PROTOCOL__context_switch_context_type_t context_type, uint16_t context_index, const std::string &context_name)
 {
     uint8_t converted_context_type = static_cast<uint8_t>(context_type);
     uint32_t action_list_base_address = 0;
@@ -521,7 +521,7 @@ Expected<ordered_json> DownloadActionListCommand::parse_network_group(Device &de
     network_group_json["contexts"].emplace_back(preliminary_context_json.release());
 
     const auto dynamic_contexts_count = number_of_dynamic_contexts_per_network_group.value()[network_group_id];
-    for (uint8_t context_index = 0; context_index < dynamic_contexts_count; context_index++) {
+    for (uint16_t context_index = 0; context_index < dynamic_contexts_count; context_index++) {
         auto context_json = parse_context(device, network_group_id,
             CONTROL_PROTOCOL__CONTEXT_SWITCH_CONTEXT_TYPE_DYNAMIC, context_index,
             fmt::format("dynamic_{}", context_index));

hailort/hailortcli/download_action_list_command.hpp

@@ -61,7 +61,7 @@ private:
         uint32_t current_buffer_offset, uint32_t *action_length, bool *is_repeated, uint8_t *num_repeated,
         CONTEXT_SWITCH_DEFS__ACTION_TYPE_t *sub_action_type, uint32_t *time_stamp);
     static Expected<ordered_json> parse_context(Device &device, uint32_t network_group_id,
-        CONTROL_PROTOCOL__context_switch_context_type_t context_type, uint8_t context_index,
+        CONTROL_PROTOCOL__context_switch_context_type_t context_type, uint16_t context_index,
         const std::string &context_name);
     static double get_accumulator_mean_value(const AccumulatorPtr &accumulator, double default_value = INVALID_NUMERIC_VALUE);
     static Expected<ordered_json> parse_network_groups(Device &device, const ConfiguredNetworkGroupVector &network_groups);

hailort/hailortcli/graph_printer.cpp

@@ -119,16 +119,16 @@ DotWriter::HtmlString PipelineGraphNode::format_runtime_stats(const std::vector<
         // We split the statistics into two lines
         std::stringstream string_stream;
         string_stream << "<B>" << accumulator->get_data_type() << ": </B>";
-        string_stream << "mean=" << InferResultsFormatUtils::format_statistic(accumulator_result.mean()) << ", ";
+        string_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.mean(), "mean") << ", ";
-        string_stream << "min=" << InferResultsFormatUtils::format_statistic(accumulator_result.min()) << ", ";
+        string_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.min(), "min") << ", ";
-        string_stream << "max=" << InferResultsFormatUtils::format_statistic(accumulator_result.max()) << ", ";
+        string_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.max(), "max") << ", ";
         lines.emplace_back(string_stream.str());
 
         // Clear the stream and format the next line
         string_stream.str("");
-        string_stream << "var=" << InferResultsFormatUtils::format_statistic(accumulator_result.var()) << ", ";
+        string_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.var(), "var") << ", ";
-        string_stream << "sd=" << InferResultsFormatUtils::format_statistic(accumulator_result.sd()) << ", ";
+        string_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.sd(), "sd") << ", ";
-        string_stream << "mean_sd=" << InferResultsFormatUtils::format_statistic(accumulator_result.mean_sd());
+        string_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.mean_sd(), "mean_sd");
         lines.emplace_back(string_stream.str());
     }
 

hailort/hailortcli/infer_stats_printer.cpp

@@ -30,27 +30,7 @@ static std::string infer_mode_to_string(InferMode infer_mode)
     }
 }
 
-std::string InferResultsFormatUtils::format_statistic(const Expected<double> &statistic, uint32_t precision)
+double InferStatsPrinter::latency_result_to_ms(std::chrono::nanoseconds latency)
-{
-    if (!statistic.has_value()) {
-        return "-";
-    }
-
-    std::stringstream string_stream;
-    string_stream << std::fixed << std::setprecision(precision) << statistic.value();
-    return string_stream.str();
-}
-
-std::string InferResultsFormatUtils::format_statistic(const Expected<size_t> &statistic)
-{
-    if (!statistic.has_value()) {
-        return "-";
-    }
-
-    return std::to_string(statistic.value());
-}
-
-double InferResultsFormatUtils::latency_result_to_ms(std::chrono::nanoseconds latency)
 {
     return std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(latency).count();
 }
@@ -172,12 +152,12 @@ void InferStatsPrinter::print_csv(const std::vector<std::string> &network_groups
             m_results_csv_file << ",";
 
             if (auto hw_latency = results.hw_latency()) {
-                m_results_csv_file << InferResultsFormatUtils::latency_result_to_ms(hw_latency.value());
+                m_results_csv_file << InferStatsPrinter::latency_result_to_ms(hw_latency.value());
             }
             m_results_csv_file << ",";
 
             if (auto overall_latency = results.overall_latency()) {
-                m_results_csv_file << InferResultsFormatUtils::latency_result_to_ms(overall_latency.value());
+                m_results_csv_file << InferStatsPrinter::latency_result_to_ms(overall_latency.value());
             }
 
             // TODO HRT-5363 support multiple devices (Currently assumes 1 device in the map)
@@ -327,12 +307,12 @@ void InferStatsPrinter::print_benchmark_csv(InferResult &hw_inference_result,
         m_results_csv_file << ",";
 
         if (auto hw_latency = latency_res->hw_latency()) {
-            m_results_csv_file << InferResultsFormatUtils::latency_result_to_ms(hw_latency.value());
+            m_results_csv_file << InferStatsPrinter::latency_result_to_ms(hw_latency.value());
         }
         m_results_csv_file << ",";
 
         if (auto overall_latency = latency_res->overall_latency()) {
-            m_results_csv_file << InferResultsFormatUtils::latency_result_to_ms(overall_latency.value());
+            m_results_csv_file << InferStatsPrinter::latency_result_to_ms(overall_latency.value());
         }
 
         // TODO HRT-5363 support multiple devices (Currently assumes 1 device in the map)
@@ -378,11 +358,11 @@ void InferStatsPrinter::print_stdout_single_element(const T &results, size_t fra
     }
 
     if (auto hw_latency = results.hw_latency()) {
-        std::cout << "    HW Latency: " << InferResultsFormatUtils::latency_result_to_ms(hw_latency.value()) << " ms" << std::endl;
+        std::cout << "    HW Latency: " << InferStatsPrinter::latency_result_to_ms(hw_latency.value()) << " ms" << std::endl;
     }
 
     if (auto overall_latency = results.overall_latency()) {
-        std::cout << "    Overall Latency: " << InferResultsFormatUtils::latency_result_to_ms(overall_latency.value()) << " ms" << std::endl;
+        std::cout << "    Overall Latency: " << InferStatsPrinter::latency_result_to_ms(overall_latency.value()) << " ms" << std::endl;
     }
 
 }
@@ -489,12 +469,12 @@ void InferStatsPrinter::write_accumulator_results(std::ofstream &output_stream,
     output_stream << vstream_name << ",";
     output_stream << accumulator->get_data_type() << ",";
     output_stream << elem_name << ",";
-    output_stream << InferResultsFormatUtils::format_statistic(accumulator_result.mean()) << ",";
+    output_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.mean()) << ",";
-    output_stream << InferResultsFormatUtils::format_statistic(accumulator_result.min()) << ",";
+    output_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.min()) << ",";
-    output_stream << InferResultsFormatUtils::format_statistic(accumulator_result.max()) << ",";
+    output_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.max()) << ",";
-    output_stream << InferResultsFormatUtils::format_statistic(accumulator_result.var()) << ",";
+    output_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.var()) << ",";
-    output_stream << InferResultsFormatUtils::format_statistic(accumulator_result.sd()) << ",";
+    output_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.sd()) << ",";
-    output_stream << InferResultsFormatUtils::format_statistic(accumulator_result.mean_sd()) << ",";
+    output_stream << AccumulatorResultsHelper::format_statistic(accumulator_result.mean_sd()) << ",";
     if (NO_INDEX != index) {
         output_stream << index;
     }

hailort/hailortcli/infer_stats_printer.hpp

@@ -15,19 +15,9 @@
 
 #include <limits>
 
-class InferResultsFormatUtils final {
-public:
-    InferResultsFormatUtils() = delete;
-
-    static const uint32_t DEFAULT_FLOATING_POINT_PRECISION = 4;
-
-    static std::string format_statistic(const Expected<double> &statistic, uint32_t precision = DEFAULT_FLOATING_POINT_PRECISION);
-    static std::string format_statistic(const Expected<size_t> &statistic);
-    static double latency_result_to_ms(std::chrono::nanoseconds latency);
-};
-
 class InferStatsPrinter final {
 public:
+    static double latency_result_to_ms(std::chrono::nanoseconds latency);
     static Expected<InferStatsPrinter> create(const inference_runner_params &params, bool print_running_info = true);
     void print(const std::vector<std::string> &network_groups_names, Expected<InferResult> &inference_result);
     void print_benchmark_csv(InferResult &hw_inference_result,

hailort/hailortcli/inference_progress.cpp

@@ -142,7 +142,7 @@ std::string NetworkProgressBar::get_progress_text()
         double avg_hw_latency = 0;
         auto latency_expected = m_configured_network_group->get_latency_measurement(m_network_name);
         if (latency_expected) {
-            avg_hw_latency = InferResultsFormatUtils::latency_result_to_ms(latency_expected.release().avg_hw_latency);
+            avg_hw_latency = InferStatsPrinter::latency_result_to_ms(latency_expected.release().avg_hw_latency);
         }
 
         if (avg_hw_latency > 0) {

hailort/hailortcli/run2/io_wrappers.hpp

@@ -16,12 +16,15 @@
 #include "common/file_utils.hpp"
 #include "common/latency_meter.hpp"
 
+#include "hailo/dma_mapped_buffer.hpp"
+
 #include <chrono>
 #include <string>
 
 using namespace hailort;
 
 constexpr uint32_t UNLIMITED_FRAMERATE = 0;
+constexpr size_t   AMOUNT_OF_OUTPUT_BUFFERS_SYNC_API = 1;
 
 #ifndef HAILO_EMULATOR
 constexpr std::chrono::milliseconds HAILORTCLI_DEFAULT_TIMEOUT(HAILO_DEFAULT_VSTREAM_TIMEOUT_MS);
@@ -44,21 +47,27 @@ private:
 };
 
 // Wrapper for InputStream or InputVStream objects.
-// We use std::enable_from_this because on async api the callback is using `this`. We want to increase the reference
+// We use std::enable_from_this because on async api, we want to increase the ref count of this object until the
-// count until the callback is over.
+// callback is called. It can happen since network_group->shutdown() may be called after this object is being
+// destructed.
 template<typename Writer>
 class WriterWrapper final : public std::enable_shared_from_this<WriterWrapper<Writer>>
 {
 public:
     template<typename WriterParams>
     static Expected<std::shared_ptr<WriterWrapper>> create(Writer &writer, const WriterParams &params,
-        const LatencyMeterPtr &overall_latency_meter, uint32_t framerate)
+        VDevice &vdevice, const LatencyMeterPtr &overall_latency_meter, uint32_t framerate, bool async_api)
     {
-        auto dataset = create_dataset(writer, params);
+        TRY(auto dataset, create_dataset(writer, params));
-        CHECK_EXPECTED(dataset);
+
+        std::vector<DmaMappedBuffer> dataset_mapped_buffers;
+        if (async_api) {
+            TRY(dataset_mapped_buffers, dma_map_dataset(dataset, vdevice));
+        }
 
         std::shared_ptr<WriterWrapper> wrapper(
-            new (std::nothrow) WriterWrapper(writer, dataset.release(), overall_latency_meter, framerate));
+            new (std::nothrow) WriterWrapper(writer, std::move(dataset), std::move(dataset_mapped_buffers),
+                                             overall_latency_meter, framerate));
         CHECK_NOT_NULL_AS_EXPECTED(wrapper, HAILO_OUT_OF_HOST_MEMORY);
 
         return wrapper;
@@ -102,10 +111,11 @@ public:
     }
 
 private:
-    WriterWrapper(Writer &writer, std::vector<BufferPtr> &&dataset, const LatencyMeterPtr &overall_latency_meter,
+    WriterWrapper(Writer &writer, std::vector<BufferPtr> &&dataset, std::vector<DmaMappedBuffer> &&dataset_mapped_buffers,
-                  uint32_t framerate) :
+                  const LatencyMeterPtr &overall_latency_meter, uint32_t framerate) :
         m_writer(std::ref(writer)),
         m_dataset(std::move(dataset)),
+        m_dataset_mapped_buffers(std::move(dataset_mapped_buffers)),
         m_overall_latency_meter(overall_latency_meter),
         m_framerate_throttle(framerate)
     {}
@@ -142,6 +152,7 @@ private:
     static Expected<std::vector<BufferPtr>> create_constant_dataset(size_t frame_size)
     {
         const uint8_t const_byte = 0xAB;
+
         auto constant_buffer = Buffer::create_shared(frame_size, const_byte, BufferStorageParams::create_dma());
         CHECK_EXPECTED(constant_buffer);
 
@@ -169,9 +180,20 @@ private:
         return dataset;
     }
 
+    static Expected<std::vector<DmaMappedBuffer>> dma_map_dataset(const std::vector<BufferPtr> &dataset, VDevice &vdevice) {
+        std::vector<DmaMappedBuffer> dataset_mapped_buffers;
+        for (const auto &buffer : dataset) {
+            auto mapped_buffer = DmaMappedBuffer::create(vdevice, buffer->data(), buffer->size(), HAILO_DMA_BUFFER_DIRECTION_H2D);
+            CHECK_EXPECTED(mapped_buffer);
+            dataset_mapped_buffers.emplace_back(mapped_buffer.release());
+        }
+        return dataset_mapped_buffers;
+    }
+
     std::reference_wrapper<Writer> m_writer;
 
     std::vector<BufferPtr> m_dataset;
+    std::vector<DmaMappedBuffer> m_dataset_mapped_buffers;
     size_t m_current_buffer_index = 0;
 
     LatencyMeterPtr m_overall_latency_meter;
@@ -182,20 +204,51 @@ template<typename Writer>
 using WriterWrapperPtr = std::shared_ptr<WriterWrapper<Writer>>;
 
 // Wrapper for OutputStream or OutputVStream objects.
-// We use std::enable_from_this because on async api the callback is using `this`. We want to increase the reference
+// We use std::enable_from_this because on async api, we want to increase the ref count of this object until the
-// count until the callback is over.
+// callback is called. It can happen since network_group->shutdown() may be called after this object is being
+// destructed.
 template<typename Reader>
 class ReaderWrapper final : public std::enable_shared_from_this<ReaderWrapper<Reader>>
 {
 public:
-    static Expected<std::shared_ptr<ReaderWrapper>> create(Reader &reader, const LatencyMeterPtr &overall_latency_meter,
+
-        std::shared_ptr<NetworkLiveTrack> net_live_track)
+    // Function that gets the amount of output buffers needed for stream. Templated for both possible cases of Types that
+    // ReaderWrapper can wrap - OutputStream and OutputVStream
+
+    // In async create amount of output buffers equal to async_max_queue_size - we do this because in async mode we want
+    // each stream to have its own buffer. (Otherwise can cause bugs in NMS Async mode.)
+    static Expected<size_t> get_amount_of_output_buffers(OutputStream &output_stream, bool async_api)
     {
-        auto buffer = Buffer::create_shared(reader.get_frame_size(), BufferStorageParams::create_dma());
+        if (async_api) {
-        CHECK_EXPECTED(buffer);
+            return output_stream.get_async_max_queue_size();
+        } else {
+            return static_cast<size_t>(AMOUNT_OF_OUTPUT_BUFFERS_SYNC_API);
+        }
+    }
+
+    // Vstreams will always be sync hence 1 output buffer is enough.
+    static Expected<size_t> get_amount_of_output_buffers(OutputVStream &output_vstream, bool async_api)
+    {
+        (void) output_vstream;
+        (void) async_api;
+        return static_cast<size_t>(AMOUNT_OF_OUTPUT_BUFFERS_SYNC_API);
+    }
+
+    static Expected<std::shared_ptr<ReaderWrapper>> create(Reader &reader, VDevice &vdevice,
+        const LatencyMeterPtr &overall_latency_meter, std::shared_ptr<NetworkLiveTrack> net_live_track, bool async_api)
+    {
+        TRY(const auto amount_of_output_buffers, get_amount_of_output_buffers(reader,async_api));
+
+        TRY(auto output_buffers, create_output_buffers(reader, amount_of_output_buffers));
+
+        std::vector<DmaMappedBuffer> dma_mapped_buffers;
+        if (async_api) {
+            TRY(dma_mapped_buffers, dma_map_output_buffers(vdevice, amount_of_output_buffers, output_buffers));
+        }
 
         std::shared_ptr<ReaderWrapper> wrapper(
-            new (std::nothrow) ReaderWrapper(reader, buffer.release(), overall_latency_meter, net_live_track));
+            new (std::nothrow) ReaderWrapper(reader, std::move(output_buffers), std::move(dma_mapped_buffers),
+            overall_latency_meter, net_live_track));
         CHECK_NOT_NULL_AS_EXPECTED(wrapper, HAILO_OUT_OF_HOST_MEMORY);
 
         return wrapper;
@@ -206,7 +259,7 @@ public:
 
     hailo_status read()
     {
-        auto status = get().read(MemoryView(*m_buffer));
+        auto status = get().read(MemoryView(*next_buffer()));
         if (HAILO_SUCCESS != status) {
             return status;
         }
@@ -217,13 +270,13 @@ public:
 
     hailo_status wait_for_async_ready()
     {
-        return get().wait_for_async_ready(m_buffer->size(), HAILORTCLI_DEFAULT_TIMEOUT);
+        return get().wait_for_async_ready(m_buffer[0]->size(), HAILORTCLI_DEFAULT_TIMEOUT);
     }
 
     hailo_status read_async(typename Reader::TransferDoneCallback callback)
     {
         auto self = std::enable_shared_from_this<ReaderWrapper<Reader>>::shared_from_this();
-        return get().read_async(MemoryView(*m_buffer),
+        return get().read_async(MemoryView(*next_buffer()),
             [self, original=callback](const typename Reader::CompletionInfo &completion_info) {
                 original(completion_info);
                 if (completion_info.status == HAILO_SUCCESS) {
@@ -233,10 +286,11 @@ public:
     }
 
 private:
-    ReaderWrapper(Reader &reader, BufferPtr &&buffer, const LatencyMeterPtr &overall_latency_meter,
+    ReaderWrapper(Reader &reader, std::vector<BufferPtr> &&buffer, std::vector<DmaMappedBuffer> &&mapped_buffer_ptr,
-                  std::shared_ptr<NetworkLiveTrack> net_live_track) :
+                  const LatencyMeterPtr &overall_latency_meter, std::shared_ptr<NetworkLiveTrack> net_live_track) :
         m_reader(std::ref(reader)),
         m_buffer(std::move(buffer)),
+        m_mapped_buffer_ptr(std::move(mapped_buffer_ptr)),
         m_overall_latency_meter(overall_latency_meter),
         m_net_live_track(net_live_track)
     {}
@@ -253,9 +307,51 @@ private:
     }
 
     std::reference_wrapper<Reader> m_reader;
-    BufferPtr m_buffer;
+    std::vector<BufferPtr> m_buffer;
+    std::vector<DmaMappedBuffer> m_mapped_buffer_ptr;
     LatencyMeterPtr m_overall_latency_meter;
     std::shared_ptr<NetworkLiveTrack> m_net_live_track;
+    size_t m_current_buffer_index = 0;
+
+    static Expected<std::vector<BufferPtr>> create_output_buffers(Reader &reader, size_t amount_of_output_buffers)
+    {
+        std::vector<BufferPtr> output_buffers;
+        output_buffers.reserve(amount_of_output_buffers);
+
+        for (size_t i = 0; i < amount_of_output_buffers; i++) {
+            TRY(auto buffer, Buffer::create_shared(reader.get_frame_size(), BufferStorageParams::create_dma()));
+            output_buffers.emplace_back(std::move(buffer));
+        }
+
+        return output_buffers;
+    }
+
+    static Expected<std::vector<DmaMappedBuffer>>dma_map_output_buffers(VDevice &vdevice, size_t amount_of_output_buffers,
+        const std::vector<BufferPtr> &output_buffers)
+    {
+        std::vector<DmaMappedBuffer> mapped_output_buffers;
+        mapped_output_buffers.reserve(amount_of_output_buffers);
+        
+        for (const auto& output_buffer : output_buffers) {
+            TRY(auto mapped_buffer,
+                DmaMappedBuffer::create(vdevice, output_buffer->data(), output_buffer->size(), HAILO_DMA_BUFFER_DIRECTION_D2H));
+            mapped_output_buffers.emplace_back(std::move(mapped_buffer));
+        }
+
+        return mapped_output_buffers;
+    }
+
+    size_t next_buffer_index()
+    {
+        const auto index = m_current_buffer_index;
+        m_current_buffer_index = (m_current_buffer_index + 1) % m_buffer.size();
+        return index;
+    }
+
+    BufferPtr next_buffer()
+    {
+        return m_buffer[next_buffer_index()];
+    }
 };
 
 template<typename Reader>

hailort/hailortcli/run2/network_live_track.cpp

@@ -75,7 +75,7 @@ uint32_t NetworkLiveTrack::push_text_impl(std::stringstream &ss)
     if (m_cng) {
         auto hw_latency_measurement = m_cng->get_latency_measurement();
         if (hw_latency_measurement) {
-            ss << fmt::format("{}hw latency: {:.2f} ms", get_separator(), InferResultsFormatUtils::latency_result_to_ms(hw_latency_measurement->avg_hw_latency));
+            ss << fmt::format("{}hw latency: {:.2f} ms", get_separator(), InferStatsPrinter::latency_result_to_ms(hw_latency_measurement->avg_hw_latency));
         } else if (HAILO_NOT_AVAILABLE != hw_latency_measurement.status()) { // HAILO_NOT_AVAILABLE is a valid error, we ignore it
             ss << fmt::format("{}hw latency: NaN (err)", get_separator());
         }
@@ -83,7 +83,7 @@ uint32_t NetworkLiveTrack::push_text_impl(std::stringstream &ss)
     else {
         auto hw_latency_measurement = m_configured_infer_model->get_hw_latency_measurement();
         if (hw_latency_measurement) {
-            ss << fmt::format("{}hw latency: {:.2f} ms", get_separator(), InferResultsFormatUtils::latency_result_to_ms(hw_latency_measurement->avg_hw_latency));
+            ss << fmt::format("{}hw latency: {:.2f} ms", get_separator(), InferStatsPrinter::latency_result_to_ms(hw_latency_measurement->avg_hw_latency));
         }
         else if (HAILO_NOT_AVAILABLE != hw_latency_measurement.status()) { // HAILO_NOT_AVAILABLE is a valid error, we ignore it
             ss << fmt::format("{}hw latency: NaN (err)", get_separator());
@@ -93,7 +93,7 @@ uint32_t NetworkLiveTrack::push_text_impl(std::stringstream &ss)
     if (m_overall_latency_meter) {
         auto overall_latency_measurement = m_overall_latency_meter->get_latency(false);
         if (overall_latency_measurement) {
-            ss << fmt::format("{}overall latency: {:.2f} ms", get_separator(), InferResultsFormatUtils::latency_result_to_ms(*overall_latency_measurement));
+            ss << fmt::format("{}overall latency: {:.2f} ms", get_separator(), InferStatsPrinter::latency_result_to_ms(*overall_latency_measurement));
         }
         else if (HAILO_NOT_AVAILABLE != overall_latency_measurement.status()) { // HAILO_NOT_AVAILABLE is a valid error, we ignore it
             ss << fmt::format("{}overall latency: NaN (err)", get_separator());
@@ -127,13 +127,13 @@ void NetworkLiveTrack::push_json_impl(nlohmann::ordered_json &json)
     if (m_cng) {
         auto hw_latency_measurement = m_cng->get_latency_measurement();
         if (hw_latency_measurement){
-            network_group_json["hw_latency"] = InferResultsFormatUtils::latency_result_to_ms(hw_latency_measurement->avg_hw_latency);
+            network_group_json["hw_latency"] = InferStatsPrinter::latency_result_to_ms(hw_latency_measurement->avg_hw_latency);
         }
     }
     else {
         auto hw_latency_measurement = m_configured_infer_model->get_hw_latency_measurement();
         if (hw_latency_measurement){
-            network_group_json["hw_latency"] = InferResultsFormatUtils::latency_result_to_ms(hw_latency_measurement->avg_hw_latency);
+            network_group_json["hw_latency"] = InferStatsPrinter::latency_result_to_ms(hw_latency_measurement->avg_hw_latency);
         }
     }
 
@@ -141,7 +141,7 @@ void NetworkLiveTrack::push_json_impl(nlohmann::ordered_json &json)
     if (m_overall_latency_meter){
         auto overall_latency_measurement = m_overall_latency_meter->get_latency(false);
         if (overall_latency_measurement){
-            network_group_json["overall_latency"] = InferResultsFormatUtils::latency_result_to_ms(*overall_latency_measurement);
+            network_group_json["overall_latency"] = InferStatsPrinter::latency_result_to_ms(*overall_latency_measurement);
         }
     }
     json["network_groups"].emplace_back(network_group_json);

hailort/hailortcli/run2/network_runner.cpp

@@ -254,7 +254,7 @@ Expected<std::shared_ptr<NetworkRunner>> NetworkRunner::create_shared(VDevice &v
 
         switch (final_net_params.mode)
         {
-        case InferenceMode::FULL:
+        case InferenceMode::FULL_SYNC:
         {
             std::map<std::string, hailo_vstream_params_t> vstreams_params;
             for (auto &vstream_params : final_net_params.vstream_params) {
@@ -263,13 +263,13 @@ Expected<std::shared_ptr<NetworkRunner>> NetworkRunner::create_shared(VDevice &v
             auto vstreams = create_vstreams(*cfgr_net_group, vstreams_params);
             CHECK_EXPECTED(vstreams);
 
-            auto net_runner = make_shared_nothrow<FullNetworkRunner>(final_net_params, expected_net_group_name.value(), vdevice,
+            auto net_runner = make_shared_nothrow<FullSyncNetworkRunner>(final_net_params, expected_net_group_name.value(), vdevice,
                 std::move(vstreams->first), std::move(vstreams->second), cfgr_net_group);
             CHECK_NOT_NULL_AS_EXPECTED(net_runner, HAILO_OUT_OF_HOST_MEMORY);
             net_runner_ptr = std::static_pointer_cast<NetworkRunner>(net_runner);
             break;
         }
-        case InferenceMode::RAW:            // Fallthrough
+        case InferenceMode::RAW_SYNC:       // Fallthrough
         case InferenceMode::RAW_ASYNC:      // Fallthrough
         case InferenceMode::RAW_ASYNC_SINGLE_THREAD:
         {
@@ -425,10 +425,10 @@ Expected<std::pair<std::vector<InputVStream>, std::vector<OutputVStream>>> Netwo
 }
 
 const std::vector<hailo_status> NetworkRunner::ALLOWED_INFERENCE_RETURN_VALUES{
-    {HAILO_SUCCESS, HAILO_STREAM_ABORTED_BY_USER, HAILO_SHUTDOWN_EVENT_SIGNALED}
+    {HAILO_SUCCESS, HAILO_STREAM_ABORT, HAILO_SHUTDOWN_EVENT_SIGNALED}
 };
 
-FullNetworkRunner::FullNetworkRunner(const NetworkParams &params, const std::string &name, VDevice &vdevice,
+FullSyncNetworkRunner::FullSyncNetworkRunner(const NetworkParams &params, const std::string &name, VDevice &vdevice,
                                      std::vector<InputVStream> &&input_vstreams, std::vector<OutputVStream> &&output_vstreams,
                                      std::shared_ptr<ConfiguredNetworkGroup> cng) :
     NetworkRunner(params, name, vdevice, cng),
@@ -437,14 +437,15 @@ FullNetworkRunner::FullNetworkRunner(const NetworkParams &params, const std::str
 {
 }
 
-Expected<std::vector<AsyncThreadPtr<hailo_status>>> FullNetworkRunner::start_inference_threads(EventPtr shutdown_event,
+Expected<std::vector<AsyncThreadPtr<hailo_status>>> FullSyncNetworkRunner::start_inference_threads(EventPtr shutdown_event,
     std::shared_ptr<NetworkLiveTrack> net_live_track)
 {
+    static const bool SYNC_API = false;
     std::vector<AsyncThreadPtr<hailo_status>> threads;
     for (auto &input_vstream : m_input_vstreams) {
         const auto vstream_params = get_params(input_vstream.name());
-        auto writer = WriterWrapper<InputVStream>::create(input_vstream, vstream_params, m_overall_latency_meter,
+        auto writer = WriterWrapper<InputVStream>::create(input_vstream, vstream_params, m_vdevice,
-            m_params.framerate);
+            m_overall_latency_meter, m_params.framerate, SYNC_API);
         CHECK_EXPECTED(writer);
 
         threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("WRITE",
@@ -455,8 +456,8 @@ Expected<std::vector<AsyncThreadPtr<hailo_status>>> FullNetworkRunner::start_inf
 
     bool first = true; //TODO: check with multiple outputs
     for (auto &output_vstream : m_output_vstreams) {
-        auto reader = ReaderWrapper<OutputVStream>::create(output_vstream, m_overall_latency_meter,
+        auto reader = ReaderWrapper<OutputVStream>::create(output_vstream, m_vdevice,
-            first ? net_live_track : nullptr);
+            m_overall_latency_meter, first ? net_live_track : nullptr, SYNC_API);
         CHECK_EXPECTED(reader);
 
         threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("READ",
@@ -469,12 +470,12 @@ Expected<std::vector<AsyncThreadPtr<hailo_status>>> FullNetworkRunner::start_inf
     return threads;
 }
 
-void FullNetworkRunner::stop()
+void FullSyncNetworkRunner::stop()
 {
     (void) m_cng->shutdown();
 }
 
-std::set<std::string> FullNetworkRunner::get_input_names()
+std::set<std::string> FullSyncNetworkRunner::get_input_names()
 {
     std::set<std::string> result;
 
@@ -485,7 +486,7 @@ std::set<std::string> FullNetworkRunner::get_input_names()
     return result;
 }
 
-std::set<std::string> FullNetworkRunner::get_output_names()
+std::set<std::string> FullSyncNetworkRunner::get_output_names()
 {
     std::set<std::string> result;
 
@@ -496,7 +497,7 @@ std::set<std::string> FullNetworkRunner::get_output_names()
     return result;
 }
 
-VStreamParams FullNetworkRunner::get_params(const std::string &name)
+VStreamParams FullSyncNetworkRunner::get_params(const std::string &name)
 {
     for (const auto &params : m_params.vstream_params) {
         if (name == params.name) {
@@ -552,9 +553,12 @@ Expected<AsyncInferJob> FullAsyncNetworkRunner::create_infer_job(const Configure
         m_overall_latency_meter->add_start_sample(std::chrono::steady_clock::now().time_since_epoch());
     }
     auto job = m_configured_infer_model->run_async(bindings, [=, &inference_status] (const AsyncInferCompletionInfo &completion_info) {
+
         if (HAILO_SUCCESS != completion_info.status) {
             inference_status = completion_info.status;
-            LOGGER__ERROR("Failed in infer async request");
+            if (HAILO_STREAM_ABORT != completion_info.status) {
+                LOGGER__ERROR("Failed in infer async request");
+            }
             return;
         }
         if (m_overall_latency_meter) {
@@ -575,36 +579,6 @@ hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shut
 {
     auto signal_event_scope_guard = SignalEventScopeGuard(*shutdown_event);
 
-    std::map<std::string, Buffer> inputs_buffer_pool;
-    const uint8_t const_byte = 0xAB;
-    for (const auto &input_name : get_input_names()) {
-        inputs_buffer_pool[input_name] = {};
-        auto input_config = m_infer_model->input(input_name);
-        CHECK_EXPECTED_AS_STATUS(input_config);
-
-        auto params = get_params(input_name);
-        if (params.input_file_path.empty()) {
-            auto constant_buffer = Buffer::create(input_config->get_frame_size(), const_byte, BufferStorageParams::create_dma());
-            CHECK_EXPECTED_AS_STATUS(constant_buffer);
-            inputs_buffer_pool[input_name] = constant_buffer.release();
-        } else {
-            auto buffer = read_binary_file(params.input_file_path, BufferStorageParams::create_dma());
-            CHECK_EXPECTED_AS_STATUS(buffer);
-            inputs_buffer_pool[input_name] = buffer.release();
-        }
-    }
-
-    std::map<std::string, Buffer> outputs_buffer_pool;
-    for (const auto &output_name : get_output_names()) {
-        outputs_buffer_pool[output_name] = {};
-        auto output_config = m_infer_model->output(output_name);
-        CHECK_EXPECTED_AS_STATUS(output_config);
-
-        auto constant_buffer = Buffer::create(output_config->get_frame_size(), 0, BufferStorageParams::create_dma());
-        CHECK_EXPECTED_AS_STATUS(constant_buffer);
-        outputs_buffer_pool[output_name] = constant_buffer.release();
-    }
-
     std::unique_ptr<ConfiguredInferModelActivationGuard> guard = nullptr;
     if (HAILO_SCHEDULING_ALGORITHM_NONE != m_params.scheduling_algorithm) {
         auto status = m_configured_infer_model->set_scheduler_threshold(m_params.scheduler_threshold);
@@ -624,24 +598,64 @@ hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shut
     auto bindings = m_configured_infer_model->create_bindings();
     CHECK_EXPECTED_AS_STATUS(bindings);
 
-    for (auto &pair : inputs_buffer_pool) {
+    std::unordered_map<std::string, Buffer> input_buffers; // Keys are inputs names
-        auto &name = pair.first;
+    std::vector<Buffer> output_buffers;
-        auto &buffer = pair.second;
+    std::vector<DmaMappedBuffer> dma_mapped_buffers;
-        bindings->input(name)->set_buffer(hailort::MemoryView(buffer));
+
+    const uint8_t const_byte = 0xAB;
+    for (const auto &name : get_input_names()) {
+        auto input_config = m_infer_model->input(name);
+        CHECK_EXPECTED_AS_STATUS(input_config);
+
+        auto params = get_params(name);
+        auto buffer = params.input_file_path.empty() ?
+            Buffer::create(input_config->get_frame_size(), const_byte, BufferStorageParams::create_dma()) :
+            read_binary_file(params.input_file_path, BufferStorageParams::create_dma());
+        CHECK_EXPECTED_AS_STATUS(buffer);
+        CHECK(0 == (buffer->size() % input_config->get_frame_size()), HAILO_INVALID_ARGUMENT,
+            "Size of data for input '{}' must be a multiple of the frame size {}. Received - {}", name, input_config->get_frame_size(), buffer->size());
+        input_buffers.emplace(name, buffer.release());
+
+        for (uint32_t i = 0; i < (input_buffers.at(name).size() % input_config->get_frame_size()); i++) {
+            auto mapped_buffer = DmaMappedBuffer::create(m_vdevice, input_buffers.at(name).data() + (i * input_config->get_frame_size()),
+                input_config->get_frame_size(), HAILO_DMA_BUFFER_DIRECTION_H2D);
+            CHECK_EXPECTED_AS_STATUS(mapped_buffer);
+            dma_mapped_buffers.emplace_back(mapped_buffer.release());
+        }
     }
-    for (auto &pair : outputs_buffer_pool) {
+
-        auto &name = pair.first;
+    for (const auto &name : get_output_names()) {
-        auto &buffer = pair.second;
+        auto output_config = m_infer_model->output(name);
-        bindings->output(name)->set_buffer(hailort::MemoryView(buffer));
+        CHECK_EXPECTED_AS_STATUS(output_config);
+
+        auto buffer = Buffer::create(output_config->get_frame_size(), 0, BufferStorageParams::create_dma());
+        CHECK_EXPECTED_AS_STATUS(buffer);
+        output_buffers.emplace_back(buffer.release());
+
+        auto mapped_buffer = DmaMappedBuffer::create(m_vdevice, output_buffers.back().data(), output_buffers.back().size(),
+            HAILO_DMA_BUFFER_DIRECTION_D2H);
+        CHECK_EXPECTED_AS_STATUS(mapped_buffer);
+        dma_mapped_buffers.emplace_back(mapped_buffer.release());
+
+        CHECK_SUCCESS(bindings->output(name)->set_buffer(MemoryView(output_buffers.back())));
     }
 
     FramerateThrottle frame_rate_throttle(m_params.framerate);
 
     AsyncInferJob last_job;
     auto inference_status = HAILO_SUCCESS;
+    uint32_t frame_id = 0;
     while (HAILO_TIMEOUT == shutdown_event->wait(std::chrono::milliseconds(0)) && (HAILO_SUCCESS == inference_status)) {
         for (uint32_t frames_in_cycle = 0; frames_in_cycle < m_params.batch_size; frames_in_cycle++) {
-            if (HAILO_SUCCESS == m_configured_infer_model->wait_for_async_ready(HAILO_INFINITE_TIMEOUT)) {
+            for (const auto &name : get_input_names()) {
+                auto input_config = m_infer_model->input(name);
+                CHECK_EXPECTED_AS_STATUS(input_config);
+                auto offset = (frame_id % (input_buffers.at(name).size() / input_config->get_frame_size())) * input_config->get_frame_size();
+                CHECK_SUCCESS(bindings->input(name)->set_buffer(MemoryView(input_buffers.at(name).data() + offset,
+                    input_config->get_frame_size())));
+            }
+            frame_id++;
+            if (HAILO_SUCCESS == m_configured_infer_model->wait_for_async_ready(DEFAULT_TRANSFER_TIMEOUT)) {
                 auto job_exp = create_infer_job(*bindings, net_live_track, frame_rate_throttle, inference_status);
                 CHECK_EXPECTED_AS_STATUS(job_exp);
                 last_job = job_exp.release();
@@ -653,6 +667,7 @@ hailo_status FullAsyncNetworkRunner::run_single_thread_async_infer(EventPtr shut
             last_job.wait(HAILO_INFINITE_TIMEOUT);
         }
     }
+    m_configured_infer_model->shutdown();
     last_job.wait(HAILO_INFINITE_TIMEOUT);
 
     return inference_status;
@@ -674,8 +689,8 @@ Expected<std::vector<AsyncThreadPtr<hailo_status>>> RawNetworkRunner::start_infe
     std::vector<AsyncThreadPtr<hailo_status>> threads;
     for (auto &input_stream : m_input_streams) {
         const auto stream_params = get_params(input_stream.get().name());
-        auto writer = WriterWrapper<InputStream>::create(input_stream.get(), stream_params, m_overall_latency_meter,
+        auto writer = WriterWrapper<InputStream>::create(input_stream.get(), stream_params, m_vdevice,
-            m_params.framerate);
+            m_overall_latency_meter, m_params.framerate, async_streams);
         CHECK_EXPECTED(writer);
 
         if (async_streams) {
@@ -693,8 +708,8 @@ Expected<std::vector<AsyncThreadPtr<hailo_status>>> RawNetworkRunner::start_infe
 
     bool first = true; //TODO: check with multiple outputs
     for (auto &output_stream : m_output_streams) {
-        auto reader = ReaderWrapper<OutputStream>::create(output_stream.get(), m_overall_latency_meter,
+        auto reader = ReaderWrapper<OutputStream>::create(output_stream.get(), m_vdevice,
-            first ? net_live_track : nullptr);
+            m_overall_latency_meter, first ? net_live_track : nullptr, async_streams);
         CHECK_EXPECTED(reader);
 
         if (async_streams) {
@@ -717,13 +732,15 @@ Expected<std::vector<AsyncThreadPtr<hailo_status>>> RawNetworkRunner::start_infe
 hailo_status RawNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_event,
     std::shared_ptr<NetworkLiveTrack> net_live_track)
 {
+    static const bool ASYNC_API = true;
+
     // Build output wrappers
     std::vector<ReaderWrapperPtr<OutputStream>> reader_wrappers;
     std::vector<SemaphorePtr> output_semaphores;
     bool is_first_output = true;
     for (auto &output_stream : m_output_streams) {
-        auto reader_wrapper = ReaderWrapper<OutputStream>::create(output_stream.get(), m_overall_latency_meter,
+        auto reader_wrapper = ReaderWrapper<OutputStream>::create(output_stream.get(), m_vdevice,
-            is_first_output ? net_live_track : nullptr);
+            m_overall_latency_meter, is_first_output ? net_live_track : nullptr, ASYNC_API);
         CHECK_EXPECTED_AS_STATUS(reader_wrapper);
         is_first_output = false;
 
@@ -731,9 +748,9 @@ hailo_status RawNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_e
         CHECK_EXPECTED_AS_STATUS(max_queue_size);
 
         auto semaphore = Semaphore::create_shared(static_cast<uint32_t>(*max_queue_size));
-        CHECK_NOT_NULL(semaphore, HAILO_OUT_OF_HOST_MEMORY);
+        CHECK_EXPECTED_AS_STATUS(semaphore);
 
-        output_semaphores.emplace_back(semaphore);
+        output_semaphores.emplace_back(semaphore.release());
         reader_wrappers.emplace_back(reader_wrapper.release());
     }
 
@@ -742,16 +759,16 @@ hailo_status RawNetworkRunner::run_single_thread_async_infer(EventPtr shutdown_e
     std::vector<SemaphorePtr> input_semaphores;
     for (auto &input_stream : m_input_streams) {
         auto writer_wrapper = WriterWrapper<InputStream>::create(input_stream.get(),
-            get_params(input_stream.get().name()), m_overall_latency_meter, m_params.framerate);
+            get_params(input_stream.get().name()), m_vdevice, m_overall_latency_meter, m_params.framerate, ASYNC_API);
         CHECK_EXPECTED_AS_STATUS(writer_wrapper);
 
         auto max_queue_size = writer_wrapper.value()->get().get_async_max_queue_size();
         CHECK_EXPECTED_AS_STATUS(max_queue_size);
 
         auto semaphore = Semaphore::create_shared(static_cast<uint32_t>(*max_queue_size));
-        CHECK_NOT_NULL(semaphore, HAILO_OUT_OF_HOST_MEMORY);
+        CHECK_EXPECTED_AS_STATUS(semaphore);
 
-        input_semaphores.emplace_back(semaphore);
+        input_semaphores.emplace_back(semaphore.release());
         writer_wrappers.emplace_back(writer_wrapper.release());
     }
 

hailort/hailortcli/run2/network_runner.hpp

@@ -37,10 +37,10 @@ constexpr std::chrono::milliseconds SYNC_EVENT_TIMEOUT(1000);
 
 
 enum class InferenceMode {
-    FULL,
+    FULL_SYNC,
     FULL_ASYNC,
 
-    RAW,
+    RAW_SYNC,
     RAW_ASYNC,
     RAW_ASYNC_SINGLE_THREAD,
 };
@@ -166,7 +166,7 @@ protected:
 
             for (auto i = 0; i < m_params.batch_size; i++) {
                 auto status = writer->write();
-                if (status == HAILO_STREAM_ABORTED_BY_USER) {
+                if (status == HAILO_STREAM_ABORT) {
                     return status;
                 }
                 CHECK_SUCCESS(status);
@@ -198,7 +198,7 @@ protected:
 
             for (auto i = 0; i < m_params.batch_size; i++) {
                 auto status = writer->wait_for_async_ready();
-                if (status == HAILO_STREAM_ABORTED_BY_USER) {
+                if (status == HAILO_STREAM_ABORT) {
                     return status;
                 }
                 CHECK_SUCCESS(status);
@@ -209,7 +209,7 @@ protected:
                             (void)sync_event->signal();
                         }
                     });
-                if (status == HAILO_STREAM_ABORTED_BY_USER) {
+                if (status == HAILO_STREAM_ABORT) {
                     return status;
                 }
                 CHECK_SUCCESS(status);
@@ -243,7 +243,7 @@ protected:
 
             for (auto i = 0; i < m_params.batch_size; i++) {
                 auto status = reader->read();
-                if (status == HAILO_STREAM_ABORTED_BY_USER) {
+                if (status == HAILO_STREAM_ABORT) {
                     return status;
                 }
                 CHECK_SUCCESS(status);
@@ -275,7 +275,7 @@ protected:
 
             for (auto i = 0; i < m_params.batch_size; i++) {
                 auto status = reader->wait_for_async_ready();
-                if (status == HAILO_STREAM_ABORTED_BY_USER) {
+                if (status == HAILO_STREAM_ABORT) {
                     return status;
                 }
                 CHECK_SUCCESS(status);
@@ -286,7 +286,7 @@ protected:
                             (void)sync_event->signal();
                         }
                     });
-                if (status == HAILO_STREAM_ABORTED_BY_USER) {
+                if (status == HAILO_STREAM_ABORT) {
                     return status;
                 }
                 CHECK_SUCCESS(status);
@@ -323,10 +323,10 @@ private:
     static Expected<BufferPtr> create_dataset_from_input_file(const std::string &file_path, size_t size);
 };
 
-class FullNetworkRunner : public NetworkRunner
+class FullSyncNetworkRunner : public NetworkRunner
 {
 public:
-    FullNetworkRunner(const NetworkParams &params, const std::string &name, VDevice &vdevice,
+    FullSyncNetworkRunner(const NetworkParams &params, const std::string &name, VDevice &vdevice,
         std::vector<InputVStream> &&input_vstreams, std::vector<OutputVStream> &&output_vstreams,
         std::shared_ptr<ConfiguredNetworkGroup> cng);
 

hailort/hailortcli/run2/run2_command.cpp

@@ -324,6 +324,7 @@ private:
 
     bool is_ethernet_device() const;
     void validate_and_set_scheduling_algorithm();
+    void validate_mode_supports_service();
 
     std::vector<NetworkParams> m_network_params;
     uint32_t m_time_to_run;
@@ -346,7 +347,6 @@ private:
     std::string m_measure_fw_actions_output_path;
 };
 
-
 Run2::Run2() : CLI::App("Run networks", "run2")
 {
     add_measure_fw_actions_subcom();
@@ -354,14 +354,17 @@ Run2::Run2() : CLI::App("Run networks", "run2")
     add_option("-t,--time-to-run", m_time_to_run, "Time to run (seconds)")
         ->default_val(DEFAULT_TIME_TO_RUN_SECONDS)
         ->check(CLI::PositiveNumber);
-    add_option("-m,--mode", m_mode, "Inference mode")
+    auto mode = add_option("-m,--mode", m_mode, "Inference mode")
         ->transform(HailoCheckedTransformer<InferenceMode>({
-            { "full", InferenceMode::FULL },
+            { "full_sync", InferenceMode::FULL_SYNC },
+            { "full", InferenceMode::FULL_SYNC, OptionVisibility::HIDDEN }, // TODO: Remove option
             { "full_async", InferenceMode::FULL_ASYNC },
-            { "raw", InferenceMode::RAW },
+            { "raw_sync", InferenceMode::RAW_SYNC },
+            { "raw", InferenceMode::RAW_SYNC, OptionVisibility::HIDDEN }, // TODO: Remove option
             { "raw_async", InferenceMode::RAW_ASYNC },
             { "raw_async_single_thread", InferenceMode::RAW_ASYNC_SINGLE_THREAD, OptionVisibility::HIDDEN }
-        }))->default_val("full");
+        }))->default_val("full_sync");
+
     add_option("-j,--json", m_stats_json_path, "If set save statistics as json to the specified path")
         ->default_val("")
         ->check(FileSuffixValidator(JSON_SUFFIX));
@@ -412,8 +415,12 @@ Run2::Run2() : CLI::App("Run networks", "run2")
         // When working with service over ip - client doesn't have access to physical devices
     }
 
+    hailo_deprecate_options(this, { std::make_shared<ValueDeprecation>(mode, "full", "full_sync"),
+        std::make_shared<ValueDeprecation>(mode, "raw", "raw_sync") }, false);
+
     parse_complete_callback([this]() {
         validate_and_set_scheduling_algorithm();
+        validate_mode_supports_service();
     });
 }
 
@@ -578,6 +585,14 @@ bool Run2::is_ethernet_device() const
     return is_valid_ip(m_device_ids[0]);
 }
 
+void Run2::validate_mode_supports_service()
+{
+    if (m_multi_process_service) {
+        PARSE_CHECK(((InferenceMode::FULL_SYNC == m_mode) || (InferenceMode::FULL_ASYNC == m_mode)),
+            "When running multi-process, only FULL_SYNC or FULL_ASYNC modes are allowed");
+    }
+}
+
 void Run2::validate_and_set_scheduling_algorithm()
 {
     if (m_scheduling_algorithm == HAILO_SCHEDULING_ALGORITHM_NONE) {
@@ -617,9 +632,9 @@ static hailo_status wait_for_threads(std::vector<AsyncThreadPtr<hailo_status>> &
     auto last_error_status = HAILO_SUCCESS;
     for (auto& thread : threads) {
         auto thread_status = thread->get();
-        if ((HAILO_SUCCESS != thread_status) && (HAILO_STREAM_ABORTED_BY_USER != thread_status)) {
+        if ((HAILO_SUCCESS != thread_status) && (HAILO_STREAM_ABORT != thread_status)) {
             last_error_status = thread_status;
-            LOGGER__ERROR("Thread failed with with status {}", thread_status);
+            LOGGER__ERROR("Thread failed with status {}", thread_status);
         }
     }
     return last_error_status;
@@ -628,12 +643,12 @@ static hailo_status wait_for_threads(std::vector<AsyncThreadPtr<hailo_status>> &
 std::string get_str_infer_mode(const InferenceMode& infer_mode)
 {
     switch(infer_mode){
-    case InferenceMode::FULL:
+    case InferenceMode::FULL_SYNC:
-        return "full";
+        return "full_sync";
     case InferenceMode::FULL_ASYNC:
         return "full_async";
-    case InferenceMode::RAW:
+    case InferenceMode::RAW_SYNC:
-        return "raw";
+        return "raw_sync";
     case InferenceMode::RAW_ASYNC:
         return "raw_async";
     case InferenceMode::RAW_ASYNC_SINGLE_THREAD:
@@ -682,8 +697,8 @@ Expected<std::unique_ptr<VDevice>> Run2::create_vdevice()
         CHECK_AS_EXPECTED(!get_multi_process_service(), HAILO_INVALID_OPERATION, "Collecting runtime data is not supported with multi process service");
         CHECK_AS_EXPECTED(get_device_count() == 1, HAILO_INVALID_OPERATION, "Collecting runtime data is not supported with multi device");
         CHECK_AS_EXPECTED(!(get_measure_hw_latency() || get_measure_overall_latency()), HAILO_INVALID_OPERATION, "Latency measurement is not allowed when collecting runtime data");
-        CHECK_AS_EXPECTED((get_mode() == InferenceMode::RAW) || (get_mode() == InferenceMode::RAW_ASYNC), HAILO_INVALID_OPERATION,
+        CHECK_AS_EXPECTED((get_mode() == InferenceMode::RAW_SYNC) || (get_mode() == InferenceMode::RAW_ASYNC), HAILO_INVALID_OPERATION,
-            "'measure-fw-actions' is only supported with '--mode=raw'. Received mode: '{}'", get_str_infer_mode(get_mode()));
+            "'measure-fw-actions' is only supported with '--mode=raw_sync' or '--mode=raw_async'. Received mode: '{}'", get_str_infer_mode(get_mode()));
     }
 
     vdevice_params.group_id = get_group_id().c_str();
@@ -725,6 +740,8 @@ Expected<std::vector<std::shared_ptr<NetworkRunner>>> Run2::init_and_run_net_run
 
     auto signal_event_scope_guard = SignalEventScopeGuard(*shutdown_event);
 
+    activation_barrier.arrive_and_wait();
+
     if (get_measure_power() || get_measure_current() || get_measure_temp()) {
         auto physical_devices = vdevice->get_physical_devices();
         CHECK_EXPECTED(physical_devices);
@@ -732,17 +749,12 @@ Expected<std::vector<std::shared_ptr<NetworkRunner>>> Run2::init_and_run_net_run
         for (auto &device : physical_devices.value()) {
             auto measurement_live_track = MeasurementLiveTrack::create_shared(device.get(), get_measure_power(),
                 get_measure_current(), get_measure_temp());
-            if (HAILO_SUCCESS != measurement_live_track.status()) {
-                activation_barrier.terminate();
-            }
             CHECK_EXPECTED(measurement_live_track);
 
             live_stats->add(measurement_live_track.release(), 2);
         }
     }
 
-    // TODO: wait for all nets before starting timer. start() should update TimerLiveTrack to start. or maybe append here but first in vector...
-    activation_barrier.arrive_and_wait();
     CHECK_SUCCESS_AS_EXPECTED(live_stats->start());
     auto status = shutdown_event->wait(get_time_to_run());
     if (HAILO_TIMEOUT != status) {

hailort/hailortcli/run_command.cpp

@@ -401,7 +401,7 @@ hailo_status send_loop(const inference_runner_params &params, SendObject &send_o
             auto status = send_object.write(MemoryView(
                 const_cast<uint8_t*>(input_buffer->data()) + offset,
                 send_object.get_frame_size()));
-            if (HAILO_STREAM_ABORTED_BY_USER == status) {
+            if (HAILO_STREAM_ABORT == status) {
                 LOGGER__DEBUG("Input stream was aborted!");
                 return status;
             }
@@ -692,7 +692,7 @@ static hailo_status run_streaming_impl(std::shared_ptr<ConfiguredNetworkGroup> c
     auto error_status = HAILO_SUCCESS;
     for (auto& result : results) {
         auto status = result->get();
-        if (HAILO_STREAM_ABORTED_BY_USER == status) {
+        if (HAILO_STREAM_ABORT == status) {
             continue;
         }
         if (HAILO_SUCCESS != status) {

hailort/hailortcli/udp_rate_limiter_command.cpp

@@ -18,7 +18,7 @@
 #define PORTS_COUNT (16)     // Should be same as HW_PACKAGE__CORE_PKG__N_AXIS_IN
 
 UdpRateLimiterCommand::UdpRateLimiterCommand (CLI::App &parent_app) :
-    Command(parent_app.add_subcommand("udp-rate-limiter", "Limit UDP rate"))
+    Command(parent_app.add_subcommand("udp-rate-limiter", "Limit the UDP rate"))
 {
     m_set_command = m_app->add_subcommand("set", "Sets the udp rate limit");
     m_set_command->add_option("--kbit-rate", m_rate_kbit_sec, "rate in Kbit/s")

hailort/libhailort/CMakeLists.txt

@@ -2,23 +2,24 @@ cmake_minimum_required(VERSION 3.0.0)
 # set(CMAKE_C_CLANG_TIDY "clang-tidy;-checks=*")
 
 set(HAILORT_MAJOR_VERSION    4)
-set(HAILORT_MINOR_VERSION    16)
+set(HAILORT_MINOR_VERSION    17)
-set(HAILORT_REVISION_VERSION 2)
+set(HAILORT_REVISION_VERSION 0)
 
 # Add the cmake folder so the modules there are found
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 
 # Generate hef-proto files using host protobuf
-protobuf_generate_cpp(PROTO_HEF_SRC PROTO_HEF_HEADR hef.proto)
+protobuf_generate_cpp(PROTO_HEF_SRC PROTO_HEF_HEADER hef.proto)
+protobuf_generate_python(PROTO_HEF_PY hef.proto) # TODO (HRT-12504): Copy hef_pb2.py to tools directory
 
-add_library(hef_proto ${PROTO_HEF_SRC} ${PROTO_HEF_HEADR})
+add_library(hef_proto ${PROTO_HEF_SRC} ${PROTO_HEF_HEADER} ${PROTO_HEF_PY})
 target_link_libraries(hef_proto libprotobuf-lite)
 set_target_properties(hef_proto PROPERTIES CXX_STANDARD 14 GENERATED TRUE POSITION_INDEPENDENT_CODE ON)
 if(CMAKE_HOST_WIN32)
     # https://github.com/protocolbuffers/protobuf/tree/master/cmake#notes-on-compiler-warnings
     target_compile_options(hef_proto PRIVATE /wd4244)
 endif()
-get_filename_component(PROTO_HEADER_DIRECTORY ${PROTO_HEF_HEADR} DIRECTORY)
+get_filename_component(PROTO_HEADER_DIRECTORY ${PROTO_HEF_HEADER} DIRECTORY)
 target_include_directories(hef_proto
     PUBLIC
     $<BUILD_INTERFACE: ${PROTO_HEADER_DIRECTORY}>

hailort/libhailort/bindings/gstreamer/CMakeLists.txt

@@ -8,7 +8,7 @@ if(NOT CMAKE_HOST_UNIX)
     message(FATAL_ERROR "Only unix hosts are supported, stopping build")
 endif()
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 # GST_PLUGIN_DEFINE needs PACKAGE to be defined
 set(GST_HAILO_PACKAGE_NAME "hailo")
@@ -19,13 +19,14 @@ pkg_search_module(GLIB REQUIRED glib-2.0)
 pkg_search_module(GSTREAMER REQUIRED gstreamer-1.0)
 pkg_search_module(GSTREAMER_BASE REQUIRED gstreamer-base-1.0)
 pkg_search_module(GSTREAMER_VIDEO REQUIRED gstreamer-video-1.0)
+pkg_search_module(GSTREAMER_PLUGINS_BASE REQUIRED gstreamer-plugins-base-1.0)
 
 add_library(gsthailo SHARED
     gst-hailo/gsthailoplugin.cpp
+    gst-hailo/sync_gsthailonet.cpp
+    gst-hailo/sync_gst_hailosend.cpp
+    gst-hailo/sync_gst_hailorecv.cpp
     gst-hailo/gsthailonet.cpp
-    gst-hailo/gsthailosend.cpp
-    gst-hailo/gsthailorecv.cpp
-    gst-hailo/gsthailonet2.cpp
     gst-hailo/gsthailodevicestats.cpp
     gst-hailo/common.cpp
     gst-hailo/network_group_handle.cpp
@@ -51,7 +52,7 @@ target_compile_options(gsthailo PRIVATE
     -DPACKAGE="${GST_HAILO_PACKAGE_NAME}")
 
 target_include_directories(gsthailo PRIVATE ${GSTREAMER_VIDEO_INCLUDE_DIRS})
-target_link_libraries(gsthailo HailoRT::libhailort ${GSTREAMER_VIDEO_LDFLAGS})
+target_link_libraries(gsthailo HailoRT::libhailort ${GSTREAMER_VIDEO_LDFLAGS} -lgstallocators-1.0)
 
 install(TARGETS gsthailo
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved.
+ * Copyright (c) 2021-2023 Hailo Technologies Ltd. All rights reserved.
  * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
  *
  * This library is free software; you can redistribute it and/or
@@ -20,113 +20,163 @@
 #ifndef _GST_HAILONET_HPP_
 #define _GST_HAILONET_HPP_
 
-#include "common.hpp"
+#pragma GCC diagnostic push
-#include "network_group_handle.hpp"
+#pragma GCC diagnostic ignored "-Wconversion"
-#include "hailo/expected.hpp"
+#include <gst/gst.h>
-#include "hailo/event.hpp"
+#pragma GCC diagnostic pop
 
-#include <atomic>
+#include <gst/base/gstqueuearray.h>
+#include <gst/video/gstvideofilter.h>
+
+#include "hailo/infer_model.hpp"
+#include "common.hpp"
+
+#include <queue>
 #include <condition_variable>
+#include <mutex>
+#include <thread>
+
+using namespace hailort;
 
 G_BEGIN_DECLS
 
-#define GST_TYPE_HAILONET (gst_hailonet_get_type())
+#define GST_TYPE_HAILO_ALLOCATOR (gst_hailo_allocator_get_type())
-#define GST_HAILONET(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HAILONET,GstHailoNet))
+#define GST_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocator))
-#define GST_HAILONET_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HAILONET,GstHailoNetClass))
+#define GST_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocatorClass))
-#define GST_IS_HAILONET(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HAILONET))
+#define GST_IS_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_ALLOCATOR))
-#define GST_IS_HAILONET_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HAILONET))
+#define GST_IS_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_ALLOCATOR))
 
-class HailoNetImpl;
+#define MIN_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE)
-struct GstHailoNet
+#define MAX_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE * 4)
+
+#define GST_HAILO_USE_DMA_BUFFER_ENV_VAR "GST_HAILO_USE_DMA_BUFFER"
+
+struct GstHailoAllocator
 {
-    GstBin parent;
+    GstAllocator parent;
-    std::unique_ptr<HailoNetImpl> impl;
+    std::unordered_map<GstMemory*, Buffer> buffers;
 };
 
-struct GstHailoNetClass
+struct GstHailoAllocatorClass
 {
-    GstBinClass parent;
+    GstAllocatorClass parent;
 };
 
+GType gst_hailo_allocator_get_type(void);
+
 struct HailoNetProperties final
 {
 public:
-    HailoNetProperties() : m_device_id(nullptr), m_hef_path(nullptr), m_network_name(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE),
+    HailoNetProperties() : m_hef_path(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE),
-        m_is_active(false), m_device_count(0), m_vdevice_key(DEFAULT_VDEVICE_KEY), m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN),
+        m_device_id(nullptr), m_device_count(0), m_vdevice_group_id(nullptr), m_is_active(false), m_pass_through(false),
-        m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS), m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD), m_scheduler_priority(HAILO_SCHEDULER_PRIORITY_NORMAL),
+        m_outputs_min_pool_size(MIN_OUTPUTS_POOL_SIZE), m_outputs_max_pool_size(MAX_OUTPUTS_POOL_SIZE),
-        m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE), m_input_format_type(HAILO_FORMAT_TYPE_AUTO),
+        m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN), m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS),
-        m_output_format_type(HAILO_FORMAT_TYPE_AUTO), m_nms_score_threshold(0), m_nms_iou_threshold(0), m_nms_max_proposals_per_class(0)
+        m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD), m_scheduler_priority(HAILO_SCHEDULER_PRIORITY_NORMAL),
-
+        m_input_format_type(HAILO_FORMAT_TYPE_AUTO), m_output_format_type(HAILO_FORMAT_TYPE_AUTO),
+        m_nms_score_threshold(0), m_nms_iou_threshold(0), m_nms_max_proposals_per_class(0), m_input_from_meta(false),
+        m_no_transform(false), m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE), m_should_force_writable(false),
+        m_vdevice_key(DEFAULT_VDEVICE_KEY)
     {}
 
-    HailoElemProperty<gchar*> m_device_id;
+    void free_strings()
+    {
+      if (m_hef_path.was_changed()) {
+        g_free(m_hef_path.get());
+      }
+      if (m_device_id.was_changed()) {
+        g_free(m_device_id.get());
+      }
+      if (m_vdevice_group_id.was_changed()) {
+        g_free(m_vdevice_group_id.get());
+      }
+    }
+
     HailoElemProperty<gchar*> m_hef_path;
-    HailoElemProperty<gchar*> m_network_name; // This property can be network group name or a network name
     HailoElemProperty<guint16> m_batch_size;
-    HailoElemProperty<gboolean> m_is_active;
+    HailoElemProperty<gchar*> m_device_id;
     HailoElemProperty<guint16> m_device_count;
-    HailoElemProperty<guint32> m_vdevice_key;
+    HailoElemProperty<gchar*> m_vdevice_group_id;
+    HailoElemProperty<gboolean> m_is_active;
+    HailoElemProperty<gboolean> m_pass_through;
+    HailoElemProperty<guint> m_outputs_min_pool_size;
+    HailoElemProperty<guint> m_outputs_max_pool_size;
     HailoElemProperty<hailo_scheduling_algorithm_t> m_scheduling_algorithm;
     HailoElemProperty<guint32> m_scheduler_timeout_ms;
     HailoElemProperty<guint32> m_scheduler_threshold;
     HailoElemProperty<guint8> m_scheduler_priority;
-    HailoElemProperty<gboolean> m_multi_process_service;
     HailoElemProperty<hailo_format_type_t> m_input_format_type;
     HailoElemProperty<hailo_format_type_t> m_output_format_type;
     HailoElemProperty<gfloat> m_nms_score_threshold;
     HailoElemProperty<gfloat> m_nms_iou_threshold;
     HailoElemProperty<guint32> m_nms_max_proposals_per_class;
+    HailoElemProperty<gboolean> m_input_from_meta;
+    HailoElemProperty<gboolean> m_no_transform;
+    HailoElemProperty<gboolean> m_multi_process_service;
+    HailoElemProperty<gboolean> m_should_force_writable;
+
+    // Deprecated
+    HailoElemProperty<guint32> m_vdevice_key;
 };
 
-class HailoNetImpl final
+typedef struct _GstHailoNet {
-{
+  GstElement element;
-public:
+  GstPad *sinkpad;
-    static Expected<std::unique_ptr<HailoNetImpl>> create(GstHailoNet *element);
+  GstPad *srcpad;
-    HailoNetImpl(GstHailoNet *element, GstElement *hailosend, GstElement *queue, GstElement *hailorecv, EventPtr was_flushed_event);
+  GstQueueArray *input_queue;
-    ~HailoNetImpl();
+  GstQueueArray *thread_queue;
+  std::atomic_uint32_t buffers_in_thread_queue;
+  std::thread thread;
+  HailoNetProperties props;
+  GstCaps *input_caps;
+  std::atomic_bool is_thread_running;
+  std::atomic_bool has_got_eos;
+  std::mutex sink_probe_change_state_mutex;
+  bool did_critical_failure_happen;
 
-    void set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec);
+  std::unique_ptr<VDevice> vdevice;
-    void get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec);
+  std::shared_ptr<InferModel> infer_model;
-    hailo_status set_hef();
+  std::shared_ptr<ConfiguredInferModel> configured_infer_model;
-    hailo_status link_elements();
+  ConfiguredInferModel::Bindings infer_bindings;
-    hailo_status configure_network_group();
+  bool is_configured;
-    hailo_status activate_hailonet();
+  std::mutex infer_mutex;
-    hailo_status abort_streams();
 
-    gboolean src_pad_event(GstEvent *event);
+  bool has_called_activate;
-    GstPadProbeReturn sink_probe();
+  std::atomic_uint32_t ongoing_frames;
-    gboolean is_active();
+  std::condition_variable flush_cv;
-    hailo_status flush();
+  std::mutex flush_mutex;
-    hailo_status signal_was_flushed_event();
 
-    hailo_status deactivate_network_group();
+  GstVideoInfo input_frame_info;
-    HailoNetProperties &get_props() {
-        return m_props;
-    }
 
-private:
+  GstHailoAllocator *allocator;
-    void init_ghost_sink();
+  GstAllocator *dma_allocator;
-    void init_ghost_src();
+  std::unordered_map<std::string, GstBufferPool*> output_buffer_pools;
-    Expected<std::string> get_network_group_name(const std::string &network_name);
+  std::unordered_map<std::string, hailo_vstream_info_t> output_vstream_infos;
 
-    hailo_status clear_vstreams();
+  std::mutex input_queue_mutex;
+  std::mutex thread_queue_mutex;
+  std::condition_variable thread_cv;
+} GstHailoNet;
 
-    static std::atomic_uint32_t m_hailonet_count;
+typedef struct _GstHailoNetClass {
-    static std::mutex m_mutex;
+  GstElementClass parent_class;
-    GstHailoNet *m_element;
+} GstHailoNetClass;
-    HailoNetProperties m_props;
+
-    std::vector<hailo_format_with_name_t> m_output_formats;
+struct TensorInfo {
-    GstElement *m_hailosend;
+    GstBuffer *buffer;
-    GstElement *m_queue;
+    GstMapInfo buffer_info;
-    GstElement *m_hailorecv;
-    std::unique_ptr<NetworkGroupHandle> m_net_group_handle;
-    bool m_was_configured;
-    bool m_has_called_activate;
-    EventPtr m_was_flushed_event;
-    GstBufferPool *m_pool;
 };
 
-GType gst_hailonet_get_type(void);
+#define GST_TYPE_HAILONET (gst_hailonet_get_type())
+#define GST_HAILONET(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HAILONET,GstHailoNet))
+#define GST_HAILONET_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HAILONET,GstHailoNetClass))
+#define GST_IS_HAILONET(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HAILONET))
+#define GST_IS_HAILONET_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HAILONET))
+
+GType gst_hailonet_get_type (void);
 
 G_END_DECLS
 

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet2.hpp

@@ -1,172 +0,0 @@
-/*
- * Copyright (c) 2021-2023 Hailo Technologies Ltd. All rights reserved.
- * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Library General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-#ifndef _GST_HAILONET2_HPP_
-#define _GST_HAILONET2_HPP_
-
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wconversion"
-#include <gst/gst.h>
-#pragma GCC diagnostic pop
-
-#include <gst/base/gstqueuearray.h>
-#include <gst/video/gstvideofilter.h>
-
-#include "hailo/infer_model.hpp"
-#include "common.hpp"
-
-#include <queue>
-#include <condition_variable>
-#include <mutex>
-#include <thread>
-
-using namespace hailort;
-
-G_BEGIN_DECLS
-
-#define GST_TYPE_HAILO_ALLOCATOR (gst_hailo_allocator_get_type())
-#define GST_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocator))
-#define GST_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_HAILO_ALLOCATOR, GstHailoAllocatorClass))
-#define GST_IS_HAILO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_HAILO_ALLOCATOR))
-#define GST_IS_HAILO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_HAILO_ALLOCATOR))
-
-#define MIN_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE)
-#define MAX_OUTPUTS_POOL_SIZE (MAX_GSTREAMER_BATCH_SIZE * 4)
-
-struct GstHailoAllocator
-{
-    GstAllocator parent;
-    std::unordered_map<GstMemory*, Buffer> buffers;
-};
-
-struct GstHailoAllocatorClass
-{
-    GstAllocatorClass parent;
-};
-
-GType gst_hailo_allocator_get_type(void);
-
-struct HailoNet2Properties final
-{
-public:
-    HailoNet2Properties() : m_hef_path(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE),
-        m_device_id(nullptr), m_device_count(0), m_vdevice_group_id(nullptr), m_is_active(false), m_pass_through(false),
-        m_outputs_min_pool_size(MIN_OUTPUTS_POOL_SIZE), m_outputs_max_pool_size(MAX_OUTPUTS_POOL_SIZE),
-        m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN), m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS),
-        m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD), m_scheduler_priority(HAILO_SCHEDULER_PRIORITY_NORMAL),
-        m_input_format_type(HAILO_FORMAT_TYPE_AUTO), m_output_format_type(HAILO_FORMAT_TYPE_AUTO),
-        m_nms_score_threshold(0), m_nms_iou_threshold(0), m_nms_max_proposals_per_class(0), m_input_from_meta(false),
-        m_no_transform(false), m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE),
-        m_vdevice_key(DEFAULT_VDEVICE_KEY)
-    {}
-
-    void free_strings()
-    {
-      if (m_hef_path.was_changed()) {
-        g_free(m_hef_path.get());
-      }
-      if (m_device_id.was_changed()) {
-        g_free(m_device_id.get());
-      }
-      if (m_vdevice_group_id.was_changed()) {
-        g_free(m_vdevice_group_id.get());
-      }
-    }
-
-    HailoElemProperty<gchar*> m_hef_path;
-    HailoElemProperty<guint16> m_batch_size;
-    HailoElemProperty<gchar*> m_device_id;
-    HailoElemProperty<guint16> m_device_count;
-    HailoElemProperty<gchar*> m_vdevice_group_id;
-    HailoElemProperty<gboolean> m_is_active;
-    HailoElemProperty<gboolean> m_pass_through;
-    HailoElemProperty<guint> m_outputs_min_pool_size;
-    HailoElemProperty<guint> m_outputs_max_pool_size;
-    HailoElemProperty<hailo_scheduling_algorithm_t> m_scheduling_algorithm;
-    HailoElemProperty<guint32> m_scheduler_timeout_ms;
-    HailoElemProperty<guint32> m_scheduler_threshold;
-    HailoElemProperty<guint8> m_scheduler_priority;
-    HailoElemProperty<hailo_format_type_t> m_input_format_type;
-    HailoElemProperty<hailo_format_type_t> m_output_format_type;
-    HailoElemProperty<gfloat> m_nms_score_threshold;
-    HailoElemProperty<gfloat> m_nms_iou_threshold;
-    HailoElemProperty<guint32> m_nms_max_proposals_per_class;
-    HailoElemProperty<gboolean> m_input_from_meta;
-    HailoElemProperty<gboolean> m_no_transform;
-    HailoElemProperty<gboolean> m_multi_process_service;
-
-    // Deprecated
-    HailoElemProperty<guint32> m_vdevice_key;
-};
-
-typedef struct _GstHailoNet2 {
-  GstElement element;
-  GstPad *sinkpad;
-  GstPad *srcpad;
-  GstQueueArray *input_queue;
-  GstQueueArray *thread_queue;
-  std::atomic_uint32_t buffers_in_thread_queue;
-  std::thread thread;
-  HailoNet2Properties props;
-  GstCaps *input_caps;
-  std::atomic_bool is_thread_running;
-  std::atomic_bool has_got_eos;
-
-  std::unique_ptr<VDevice> vdevice;
-  std::shared_ptr<InferModel> infer_model;
-  std::shared_ptr<ConfiguredInferModel> configured_infer_model;
-  ConfiguredInferModel::Bindings infer_bindings;
-  bool is_configured;
-  std::mutex infer_mutex;
-
-  bool has_called_activate;
-  std::atomic_uint32_t ongoing_frames;
-  std::condition_variable flush_cv;
-  std::mutex flush_mutex;
-
-  GstVideoInfo input_frame_info;
-
-  GstHailoAllocator *allocator;
-  std::unordered_map<std::string, GstBufferPool*> output_buffer_pools;
-  std::unordered_map<std::string, hailo_vstream_info_t> output_vstream_infos;
-
-  std::mutex input_queue_mutex;
-  std::mutex thread_queue_mutex;
-  std::condition_variable thread_cv;
-} GstHailoNet2;
-
-typedef struct _GstHailoNet2Class {
-  GstElementClass parent_class;
-} GstHailoNet2Class;
-
-#define GST_TYPE_HAILONET2 (gst_hailonet2_get_type())
-#define GST_HAILONET2(obj) \
-  (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HAILONET2,GstHailoNet2))
-#define GST_HAILONET2_CLASS(klass) \
-  (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HAILONET2,GstHailoNet2Class))
-#define GST_IS_HAILONET2(obj) \
-  (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HAILONET2))
-#define GST_IS_HAILONET2_CLASS(klass) \
-  (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HAILONET2))
-
-GType gst_hailonet2_get_type (void);
-
-G_END_DECLS
-
-#endif /* _GST_HAILONET2_HPP_ */

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailoplugin.cpp

@@ -17,10 +17,10 @@
  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  * Boston, MA 02110-1301, USA.
  */
+#include "sync_gsthailonet.hpp"
+#include "sync_gst_hailosend.hpp"
+#include "sync_gst_hailorecv.hpp"
 #include "gsthailonet.hpp"
-#include "gsthailosend.hpp"
-#include "gsthailorecv.hpp"
-#include "gsthailonet2.hpp"
 #include "gsthailodevicestats.hpp"
 #include "metadata/tensor_meta.hpp"
 
@@ -29,11 +29,11 @@ static gboolean plugin_init(GstPlugin *plugin)
     (void)gst_tensor_meta_get_info();
     (void)gst_tensor_meta_api_get_type();
 
-    return gst_element_register(plugin, "hailonet", GST_RANK_PRIMARY, GST_TYPE_HAILONET) &&
+    return gst_element_register(plugin, "synchailonet", GST_RANK_PRIMARY, GST_TYPE_SYNC_HAILONET) &&
         gst_element_register(plugin, "hailodevicestats", GST_RANK_PRIMARY, GST_TYPE_HAILODEVICESTATS) &&
         gst_element_register(nullptr, "hailosend", GST_RANK_PRIMARY, GST_TYPE_HAILOSEND) &&
         gst_element_register(nullptr, "hailorecv", GST_RANK_PRIMARY, GST_TYPE_HAILORECV) &&
-        gst_element_register(plugin, "hailonet2", GST_RANK_PRIMARY, GST_TYPE_HAILONET2);
+        gst_element_register(plugin, "hailonet", GST_RANK_PRIMARY, GST_TYPE_HAILONET);
 }
 
 GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, hailo, "hailo gstreamer plugin", plugin_init, VERSION,

hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.cpp

@@ -302,7 +302,7 @@ Expected<std::shared_ptr<ConfiguredNetworkGroup>> NetworkGroupConfigManager::con
         GST_CHECK_EXPECTED(infos, element, RESOURCE, "Failed getting network infos");
         if ((infos.release().size() > 1) || (scheduling_algorithm == HAILO_SCHEDULING_ALGORITHM_NONE)) {
             // If cng was already configured
-            // But hailonet is not running all networks in the cng (or if not using scheduler) -
+            // But sync_hailonet is not running all networks in the cng (or if not using scheduler) -
             // Do not use multiplexer!
             return found_cng;
         }

hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.hpp

@@ -30,7 +30,7 @@
 
 using device_id_t = std::string;
 using network_name_t = std::string;
-using hailonet_name_t = std::string;
+using sync_hailonet_name_t = std::string;
 
 class NetworkGroupConfigManager final
 {
@@ -52,7 +52,7 @@ private:
 
     // TODO: change this map to store only the shared network_groups (used by multiple hailonets with the same vdevices)
     std::unordered_map<std::string, std::weak_ptr<ConfiguredNetworkGroup>> m_configured_net_groups;
-    std::unordered_map<device_id_t, std::unordered_map<network_name_t, hailonet_name_t>> m_configured_networks;
+    std::unordered_map<device_id_t, std::unordered_map<network_name_t, sync_hailonet_name_t>> m_configured_networks;
     std::mutex m_mutex;
 };
 

hailort/libhailort/bindings/gstreamer/gst-hailo/sync_gst_hailorecv.cpp

@@ -17,8 +17,8 @@
  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  * Boston, MA 02110-1301, USA.
  */
-#include "gsthailorecv.hpp"
+#include "sync_gst_hailorecv.hpp"
-#include "gsthailonet.hpp"
+#include "sync_gsthailonet.hpp"
 #include "common.hpp"
 #include "network_group_handle.hpp"
 #include "metadata/hailo_buffer_flag_meta.hpp"
@@ -182,7 +182,7 @@ GstFlowReturn HailoRecvImpl::handle_frame(GstVideoFilter */*filter*/, GstVideoFr
         switch (meta->flag) {
         case BUFFER_FLAG_FLUSH:
         {
-            hailo_status status = GST_HAILONET(GST_ELEMENT_PARENT(m_element))->impl->signal_was_flushed_event();
+            hailo_status status = GST_SYNC_HAILONET(GST_ELEMENT_PARENT(m_element))->impl->signal_was_flushed_event();
             GST_CHECK(HAILO_SUCCESS == status, GST_FLOW_ERROR, m_element, RESOURCE, "Signalling was flushed event has failed, status = %d", status);
             return GST_BASE_TRANSFORM_FLOW_DROPPED;
         }
@@ -195,7 +195,7 @@ GstFlowReturn HailoRecvImpl::handle_frame(GstVideoFilter */*filter*/, GstVideoFr
         }
     }
 
-    if (!GST_HAILONET(GST_ELEMENT_PARENT(m_element))->impl->is_active()) {
+    if (!GST_SYNC_HAILONET(GST_ELEMENT_PARENT(m_element))->impl->is_active()) {
         return GST_FLOW_OK;
     }
 
@@ -235,7 +235,7 @@ hailo_status HailoRecvImpl::read_from_vstreams(bool should_print_latency)
             GST_DEBUG("%s latency: %f milliseconds", output_info.vstream().name().c_str(), latency.count());
         }
         gst_buffer_unmap(*buffer, &buffer_info);
-        if (HAILO_STREAM_ABORTED_BY_USER == status) {
+        if (HAILO_STREAM_ABORT == status) {
             return status;
         }
         GST_CHECK_SUCCESS(status, m_element, STREAM, "Reading from vstream failed, status = %d", status);

hailort/libhailort/bindings/gstreamer/gst-hailo/sync_gst_hailosend.cpp

@@ -17,8 +17,8 @@
  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  * Boston, MA 02110-1301, USA.
  */
-#include "gsthailosend.hpp"
+#include "sync_gst_hailosend.hpp"
-#include "gsthailonet.hpp"
+#include "sync_gsthailonet.hpp"
 #include "metadata/hailo_buffer_flag_meta.hpp"
 
 #include <chrono>
@@ -87,7 +87,7 @@ Expected<std::unique_ptr<HailoSendImpl>> HailoSendImpl::create(GstHailoSend *ele
     return ptr;
 }
 
-HailoSendImpl::HailoSendImpl(GstHailoSend *element) : m_element(element), m_hailonet(nullptr), m_props(),
+HailoSendImpl::HailoSendImpl(GstHailoSend *element) : m_element(element), m_sync_hailonet(nullptr), m_props(),
     m_batch_size(HAILO_DEFAULT_BATCH_SIZE), m_last_frame_pts(0)
 {
     GST_DEBUG_CATEGORY_INIT(gst_hailosend_debug_category, "hailosend", 0, "debug category for hailosend element");
@@ -136,13 +136,14 @@ GstFlowReturn HailoSendImpl::handle_frame(GstVideoFilter */*filter*/, GstVideoFr
     assert(nullptr != frame);
     m_last_frame_pts = GST_BUFFER_TIMESTAMP(frame->buffer);
 
-    if (!GST_HAILONET(GST_ELEMENT_PARENT(m_element))->impl->is_active()) {
+    if (!GST_SYNC_HAILONET(GST_ELEMENT_PARENT(m_element))->impl->is_active()) {
         GstHailoBufferFlagMeta *meta = GST_HAILO_BUFFER_FLAG_META_ADD(frame->buffer);
         meta->flag = BUFFER_FLAG_SKIP;
         return GST_FLOW_OK;
     }
 
     hailo_pix_buffer_t pix_buffer = {};
+    pix_buffer.memory_type = HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR;
     pix_buffer.index = 0;
     pix_buffer.number_of_planes = GST_VIDEO_INFO_N_PLANES(&frame->info);
     for (uint32_t plane_index = 0; plane_index < pix_buffer.number_of_planes; plane_index++) {
@@ -174,7 +175,7 @@ hailo_status HailoSendImpl::write_to_vstreams(const hailo_pix_buffer_t &pix_buff
 {
     for (auto &in_vstream : m_input_vstreams) {
         auto status = in_vstream.write(pix_buffer);
-        if (HAILO_STREAM_ABORTED_BY_USER == status) {
+        if (HAILO_STREAM_ABORT == status) {
             return status;
         }
         GST_CHECK_SUCCESS(status, m_element, STREAM, "Failed writing to input vstream %s, status = %d", in_vstream.name().c_str(), status);
@@ -201,9 +202,9 @@ GstCaps *HailoSendImpl::get_caps(GstBaseTransform */*trans*/, GstPadDirection /*
 
     if (0 == m_input_vstream_infos.size()) {
         // Init here because it is guaranteed that we have a parent element
-        m_hailonet = GST_HAILONET(GST_ELEMENT_PARENT(m_element));
+        m_sync_hailonet = GST_SYNC_HAILONET(GST_ELEMENT_PARENT(m_element));
 
-        hailo_status status = m_hailonet->impl->set_hef();
+        hailo_status status = m_sync_hailonet->impl->set_hef();
         if (HAILO_SUCCESS != status) {
             return NULL;
         }

hailort/libhailort/bindings/gstreamer/gst-hailo/sync_gst_hailosend.hpp

@@ -22,7 +22,7 @@
 
 #include "common.hpp"
 #include "network_group_handle.hpp"
-#include "gsthailonet.hpp"
+#include "sync_gsthailonet.hpp"
 
 #include <gst/video/video.h>
 #include <gst/video/gstvideofilter.h>
@@ -92,7 +92,7 @@ private:
     hailo_status write_to_vstreams(const hailo_pix_buffer_t &pix_buffer);
     
     GstHailoSend *m_element;
-    GstHailoNet *m_hailonet;
+    GstSyncHailoNet *m_sync_hailonet;
     HailoSendProperties m_props;
     std::vector<hailo_vstream_info_t> m_input_vstream_infos;
     uint32_t m_batch_size;

hailort/libhailort/bindings/gstreamer/gst-hailo/sync_gsthailonet.cpp

@@ -0,0 +1,992 @@
+/*
+ * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#include "sync_gsthailonet.hpp"
+#include "sync_gst_hailosend.hpp"
+#include "sync_gst_hailorecv.hpp"
+#include "hailo_events/hailo_events.hpp"
+#include "metadata/hailo_buffer_flag_meta.hpp"
+#include "hailo/hailort_common.hpp"
+#include "hailo/hailort_defaults.hpp"
+
+#include <sstream>
+#include <algorithm>
+
+GST_DEBUG_CATEGORY_STATIC(gst_sync_hailonet_debug_category);
+#define GST_CAT_DEFAULT gst_sync_hailonet_debug_category
+
+constexpr std::chrono::milliseconds WAIT_FOR_FLUSH_TIMEOUT_MS(1000);
+
+static void gst_sync_hailonet_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec);
+static void gst_sync_hailonet_get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec);
+static gboolean gst_hailorecv_src_pad_event(GstPad *pad, GstObject *parent, GstEvent *event);
+static GstPadProbeReturn gst_sync_hailonet_sink_probe(GstPad *pad, GstPadProbeInfo *info, gpointer user_data);
+static GstStateChangeReturn gst_sync_hailonet_change_state(GstElement *element, GstStateChange transition);
+static void gst_sync_hailonet_flush_callback(GstSyncHailoNet *hailonet, gpointer data);
+static void gst_sync_hailonet_inner_queue_overrun_callback(GstElement *queue, gpointer udata);
+static void gst_sync_hailonet_inner_queue_underrun_callback(GstElement *queue, gpointer udata);
+
+enum
+{
+    PROP_0,
+    PROP_DEBUG,
+    PROP_DEVICE_ID,
+    PROP_HEF_PATH,
+    PROP_NETWORK_NAME,
+    PROP_BATCH_SIZE,
+    PROP_OUTPUTS_MIN_POOL_SIZE,
+    PROP_OUTPUTS_MAX_POOL_SIZE,
+    PROP_IS_ACTIVE,
+    PROP_DEVICE_COUNT,
+    PROP_VDEVICE_KEY,
+    PROP_SCHEDULING_ALGORITHM,
+    PROP_SCHEDULER_TIMEOUT_MS,
+    PROP_SCHEDULER_THRESHOLD,
+    PROP_SCHEDULER_PRIORITY,
+    PROP_MULTI_PROCESS_SERVICE,
+    PROP_INPUT_FORMAT_TYPE,
+    PROP_OUTPUT_FORMAT_TYPE,
+    PROP_NMS_SCORE_THRESHOLD,
+    PROP_NMS_IOU_THRESHOLD,
+    PROP_NMS_MAX_PROPOSALS_PER_CLASS,
+};
+
+G_DEFINE_TYPE(GstSyncHailoNet, gst_sync_hailonet, GST_TYPE_BIN);
+
+static void gst_sync_hailonet_class_init(GstSyncHailoNetClass *klass)
+{
+    GObjectClass *gobject_class = G_OBJECT_CLASS(klass);
+    GstElementClass *element_class = GST_ELEMENT_CLASS(klass);
+
+    GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY);
+    gst_element_class_add_pad_template(element_class, gst_static_pad_template_get(&src_template));
+
+    GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY);
+    gst_element_class_add_pad_template(element_class, gst_static_pad_template_get(&sink_template));
+
+    gst_element_class_set_static_metadata(element_class,
+        "sync hailonet element", "Hailo/Network",
+        "Configure and Activate Hailo Network. "
+            "Supports the \"flush\" signal which blocks until there are no buffers currently processesd in the element. "
+            "When deactivating a sync hailonet during runtime (via set_property of \"is-active\" to False), make sure that no frames are being pushed into the "
+            "hailonet, since this operation waits until there are no frames coming in.",
+        PLUGIN_AUTHOR);
+
+    element_class->change_state = GST_DEBUG_FUNCPTR(gst_sync_hailonet_change_state);
+    
+    gobject_class->set_property = gst_sync_hailonet_set_property;
+    gobject_class->get_property = gst_sync_hailonet_get_property;
+    g_object_class_install_property(gobject_class, PROP_DEBUG,
+        g_param_spec_boolean("debug", "Debug flag", "Should print debug information", false,
+            (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_DEVICE_ID,
+        g_param_spec_string("device-id", "Device ID", "Device ID ([<domain>]:<bus>:<device>.<func>, same as in lspci command). Excludes device-count.", NULL,
+            (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_DEVICE_COUNT,
+        g_param_spec_uint("device-count", "Number of devices to use", "Number of physical devices to use. Excludes device-id.", HAILO_DEFAULT_DEVICE_COUNT,
+            std::numeric_limits<uint16_t>::max(), HAILO_DEFAULT_DEVICE_COUNT, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_VDEVICE_KEY,
+        g_param_spec_uint("vdevice-key",
+            "Indicate whether to re-use or re-create vdevice",
+            "Relevant only when 'device-count' is passed. If not passed, the created vdevice will be unique to this hailonet." \
+            "if multiple hailonets share 'vdevice-key' and 'device-count', the created vdevice will be shared between those hailonets",
+            MIN_VALID_VDEVICE_KEY, std::numeric_limits<uint32_t>::max(), MIN_VALID_VDEVICE_KEY, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_HEF_PATH,
+        g_param_spec_string("hef-path", "HEF Path Location", "Location of the HEF file to read", NULL,
+            (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_NETWORK_NAME,
+        g_param_spec_string("net-name", "Network Name",
+            "Configure and run this specific network. "
+            "If not passed, configure and run the default network - ONLY if there is one network in the HEF!", NULL,
+            (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_BATCH_SIZE,
+        g_param_spec_uint("batch-size", "Inference Batch", "How many frame to send in one batch", MIN_GSTREAMER_BATCH_SIZE, MAX_GSTREAMER_BATCH_SIZE, HAILO_DEFAULT_BATCH_SIZE,
+            (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_OUTPUTS_MIN_POOL_SIZE,
+        g_param_spec_uint("outputs-min-pool-size", "Outputs Minimun Pool Size", "The minimum amount of buffers to allocate for each output layer",
+            0, std::numeric_limits<uint32_t>::max(), DEFAULT_OUTPUTS_MIN_POOL_SIZE, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_OUTPUTS_MAX_POOL_SIZE,
+        g_param_spec_uint("outputs-max-pool-size", "Outputs Maximum Pool Size",
+            "The maximum amount of buffers to allocate for each output layer or 0 for unlimited", 0, std::numeric_limits<uint32_t>::max(),
+            DEFAULT_OUTPUTS_MAX_POOL_SIZE, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_IS_ACTIVE,
+        g_param_spec_boolean("is-active", "Is Network Activated", "Controls whether this element should be active. "
+            "By default, the hailonet element will not be active unless it is the only one. "
+            "Setting this property in combination with 'scheduling-algorithm' different than HAILO_SCHEDULING_ALGORITHM_NONE is not supported.", false,
+        (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+    g_object_class_install_property(gobject_class, PROP_SCHEDULING_ALGORITHM,
+        g_param_spec_enum("scheduling-algorithm", "Scheduling policy for automatic network group switching", "Controls the Model Scheduler algorithm of HailoRT. "
+            "Gets values from the enum GstHailoSchedulingAlgorithms. "
+            "Using Model Scheduler algorithm different than HAILO_SCHEDULING_ALGORITHM_NONE, excludes the property 'is-active'. "
+            "When using the same VDevice across multiple hailonets, all should have the same 'scheduling-algorithm'. ",
+            GST_TYPE_SCHEDULING_ALGORITHM, HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN,
+        (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_SCHEDULER_TIMEOUT_MS,
+        g_param_spec_uint("scheduler-timeout-ms", "Timeout for for scheduler in ms", "The maximum time period that may pass before getting run time from the scheduler,"
+            " as long as at least one send request has been sent.",
+            HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS, std::numeric_limits<uint32_t>::max(), HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_SCHEDULER_THRESHOLD,
+        g_param_spec_uint("scheduler-threshold", "Frames threshold for scheduler", "The minimum number of send requests required before the hailonet is considered ready to get run time from the scheduler.",
+            HAILO_DEFAULT_SCHEDULER_THRESHOLD, std::numeric_limits<uint32_t>::max(), HAILO_DEFAULT_SCHEDULER_THRESHOLD, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_SCHEDULER_PRIORITY,
+        g_param_spec_uint("scheduler-priority", "Priority index for scheduler", "When the scheduler will choose the next hailonet to run, higher priority will be prioritized in the selection. "
+            "Bigger number represent higher priority",
+            HAILO_SCHEDULER_PRIORITY_MIN, HAILO_SCHEDULER_PRIORITY_MAX, HAILO_SCHEDULER_PRIORITY_NORMAL, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_MULTI_PROCESS_SERVICE,
+        g_param_spec_boolean("multi-process-service", "Should run over HailoRT service", "Controls wether to run HailoRT over its service. "
+            "To use this property, the service should be active and scheduling-algorithm should be set. Defaults to false.",
+            HAILO_DEFAULT_MULTI_PROCESS_SERVICE, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_INPUT_FORMAT_TYPE,
+        g_param_spec_enum("input-format-type", "Input format type", "Input format type(auto, float32, uint16, uint8). Default value is auto."
+            "Gets values from the enum GstHailoFormatType. ",
+            GST_TYPE_HAILO_FORMAT_TYPE, HAILO_FORMAT_TYPE_AUTO,
+        (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_OUTPUT_FORMAT_TYPE,
+        g_param_spec_enum("output-format-type", "Output format type", "Output format type(auto, float32, uint16, uint8). Default value is auto."
+            "Gets values from the enum GstHailoFormatType. ",
+            GST_TYPE_HAILO_FORMAT_TYPE, HAILO_FORMAT_TYPE_AUTO,
+        (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_NMS_SCORE_THRESHOLD,
+        g_param_spec_float("nms-score-threshold", "NMS score threshold", "Threshold used for filtering out candidates. Any box with score<TH is suppressed.",
+            0, 1, 0, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_NMS_IOU_THRESHOLD,
+        g_param_spec_float("nms-iou-threshold", "NMS IoU threshold", "Intersection over union overlap Threshold, used in the NMS iterative elimination process where potential duplicates of detected items are suppressed.",
+            0, 1, 0, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_NMS_MAX_PROPOSALS_PER_CLASS,
+        g_param_spec_uint("nms-max-proposals-per-class", "NMS max proposals per class", "Set a limit for the maximum number of boxes per class.",
+            0, std::numeric_limits<uint32_t>::max(), 0, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+    // See information about the "flush" signal in the element description
+    g_signal_new(
+        "flush",
+        GST_TYPE_SYNC_HAILONET,
+        G_SIGNAL_ACTION,
+        0, nullptr, nullptr, nullptr, G_TYPE_NONE, 0
+    );
+}
+
+std::string create_name(std::string prefix, uint32_t id)
+{
+    return prefix + std::to_string(id);
+}
+
+Expected<std::unique_ptr<HailoSyncNetImpl>> HailoSyncNetImpl::create(GstSyncHailoNet *element)
+{
+    if (nullptr == element) {
+        return make_unexpected(HAILO_INVALID_ARGUMENT);
+    }
+
+    auto hailosend_name = create_name("hailosend", HailoSyncNetImpl::m_sync_hailonet_count);
+    GstElement *hailosend = gst_element_factory_make("hailosend", hailosend_name.c_str());
+    if (nullptr == hailosend) {
+        GST_ELEMENT_ERROR(element, RESOURCE, FAILED, ("Failed creating hailosend element in bin!"), (NULL));
+        return make_unexpected(HAILO_INTERNAL_FAILURE);
+    }
+
+    g_object_set(hailosend, "qos", FALSE, NULL);
+
+    auto hailoqueue_name = create_name("hailoqueue", HailoSyncNetImpl::m_sync_hailonet_count);
+    GstElement *queue = gst_element_factory_make("queue", hailoqueue_name.c_str());
+    if (nullptr == queue) {
+        GST_ELEMENT_ERROR(element, RESOURCE, FAILED, ("Failed creating queue element in bin!"), (NULL));
+        gst_object_unref(hailosend);
+        return make_unexpected(HAILO_INTERNAL_FAILURE);
+    }
+
+    // Passing 0 disables the features here
+    g_object_set(queue, "max-size-time", (guint64)0, NULL);
+    g_object_set(queue, "max-size-bytes", (guint)0, NULL);
+    g_signal_connect(queue, "overrun", G_CALLBACK(gst_sync_hailonet_inner_queue_overrun_callback), nullptr);
+    g_signal_connect(queue, "underrun", G_CALLBACK(gst_sync_hailonet_inner_queue_underrun_callback), nullptr);
+
+    auto hailorecv_name = create_name("hailorecv", HailoSyncNetImpl::m_sync_hailonet_count);
+    GstElement *hailorecv = gst_element_factory_make("hailorecv", hailorecv_name.c_str());
+    if (nullptr == hailorecv) {
+        GST_ELEMENT_ERROR(element, RESOURCE, FAILED, ("Failed creating hailorecv element in bin!"), (NULL));
+        gst_object_unref(hailosend);
+        gst_object_unref(queue);
+        return make_unexpected(HAILO_INTERNAL_FAILURE);
+    }
+
+    g_object_set(hailorecv, "qos", FALSE, NULL);
+
+    g_signal_connect(element, "flush", G_CALLBACK(gst_sync_hailonet_flush_callback), nullptr);
+
+    auto was_flushed_event = Event::create_shared(Event::State::not_signalled);
+    GST_CHECK_EXPECTED(was_flushed_event, element, RESOURCE, "Failed allocating memory for event!");
+
+    auto ptr = make_unique_nothrow<HailoSyncNetImpl>(element, hailosend, queue, hailorecv, was_flushed_event.release());
+    if (nullptr == ptr) {
+        return make_unexpected(HAILO_OUT_OF_HOST_MEMORY);
+    }
+
+    return ptr;
+}
+
+std::atomic_uint32_t HailoSyncNetImpl::m_sync_hailonet_count(0);
+std::mutex HailoSyncNetImpl::m_mutex;
+HailoSyncNetImpl::HailoSyncNetImpl(GstSyncHailoNet *element, GstElement *hailosend, GstElement *queue, GstElement *hailorecv, EventPtr was_flushed_event) :
+    m_element(element), m_props(), m_output_formats(), m_hailosend(hailosend), m_queue(queue), m_hailorecv(hailorecv),
+    m_net_group_handle(nullptr), m_was_configured(false), m_has_called_activate(false),
+    m_was_flushed_event(was_flushed_event), m_pool(nullptr)
+{
+    GST_DEBUG_CATEGORY_INIT(gst_sync_hailonet_debug_category, "sync hailonet", 0, "debug category for sync hailonet element");
+
+    /* gst_bin_add_many cannot fail. I use this function because the elements are created here and does not come from the outside so,
+     * gst_bin_add will not fail */
+    gst_bin_add_many(GST_BIN(m_element), m_hailosend, m_queue, m_hailorecv, NULL);
+    init_ghost_sink();
+    init_ghost_src();
+
+    ++m_sync_hailonet_count;
+}
+
+HailoSyncNetImpl::~HailoSyncNetImpl()
+{
+    if (nullptr != m_pool) {
+        (void)gst_buffer_pool_set_active(m_pool, FALSE);
+    }
+}
+
+void HailoSyncNetImpl::init_ghost_sink()
+{
+    GstPad *pad = gst_element_get_static_pad(m_hailosend, "sink");
+
+    GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY);
+    GstPadTemplate *pad_tmpl = gst_static_pad_template_get(&sink_template);
+
+    GstPad *ghost_pad = gst_ghost_pad_new_from_template("sink", pad, pad_tmpl);
+    gst_pad_set_active(ghost_pad, TRUE);
+    gst_element_add_pad(GST_ELEMENT(m_element), ghost_pad);
+
+    gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_BUFFER, static_cast<GstPadProbeCallback>(gst_sync_hailonet_sink_probe), nullptr, nullptr);
+
+    gst_object_unref(pad_tmpl);
+    gst_object_unref(pad);
+}
+
+void HailoSyncNetImpl::init_ghost_src()
+{
+    GstPad *pad = gst_element_get_static_pad(m_hailorecv, "src");
+
+    GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY);
+    GstPadTemplate *pad_tmpl = gst_static_pad_template_get(&src_template);
+
+    GstPad *ghost_pad = gst_ghost_pad_new_from_template("src", pad, pad_tmpl);
+    gst_pad_set_active(ghost_pad, TRUE);
+    gst_element_add_pad(GST_ELEMENT(m_element), ghost_pad);
+
+    gst_pad_set_event_function(pad, gst_hailorecv_src_pad_event);
+
+    gst_object_unref(pad_tmpl);
+    gst_object_unref(pad);
+}
+
+void HailoSyncNetImpl::set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec)
+{
+    GST_DEBUG_OBJECT(m_element, "set_property");
+
+    if ((object == nullptr) || (value == nullptr) || (pspec == nullptr)) {
+        g_error("set_property got null parameter!");
+        return;
+    }
+
+    switch (property_id) {
+    case PROP_DEBUG:
+    {
+        gboolean debug = g_value_get_boolean(value);
+        g_object_set(m_hailosend, "debug", debug, NULL);
+        g_object_set(m_hailorecv, "debug", debug, NULL);
+        break;
+    }
+    case PROP_DEVICE_ID:
+        if (0 != m_props.m_device_count.get()) {
+            g_error("device-id and device-count excludes eachother. received device-id=%s, device-count=%d",
+                g_value_get_string(value), m_props.m_device_count.get());
+            break;
+        }
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the device ID will not take place!");
+            break;
+        }
+        if (nullptr != m_props.m_device_id.get()) {
+            g_free(m_props.m_device_id.get());
+        }
+        m_props.m_device_id = g_strdup(g_value_get_string(value));
+        break;
+    case PROP_DEVICE_COUNT:
+        if (nullptr != m_props.m_device_id.get()) {
+            g_error("device-id and device-count excludes eachother. received device-id=%s, device-count=%d",
+                m_props.m_device_id.get(), g_value_get_uint(value));
+            break;
+        }
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the device count will not take place!");
+            break;
+        }
+        m_props.m_device_count = static_cast<guint16>(g_value_get_uint(value));
+        break;
+    case PROP_VDEVICE_KEY:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the vdevice key will not take place!");
+            break;
+        }
+        m_props.m_vdevice_key = static_cast<guint32>(g_value_get_uint(value));
+        break;
+    case PROP_HEF_PATH:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the HEF path will not take place!");
+            break;
+        }
+        if (nullptr != m_props.m_hef_path.get()) {
+            g_free(m_props.m_hef_path.get());
+        }
+        m_props.m_hef_path = g_strdup(g_value_get_string(value));
+        break;
+    case PROP_NETWORK_NAME:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the network name will not take place!");
+            break;
+        }
+        if (nullptr != m_props.m_network_name.get()) {
+            g_free(m_props.m_network_name.get());
+        }
+        m_props.m_network_name = g_strdup(g_value_get_string(value));
+        break;
+    case PROP_BATCH_SIZE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the batch size will not take place!");
+            break;
+        }
+        m_props.m_batch_size = static_cast<guint16>(g_value_get_uint(value));
+        break;
+    case PROP_OUTPUTS_MIN_POOL_SIZE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the outputs minimum pool size will not take place!");
+            break;
+        }
+        g_object_set(m_hailorecv, "outputs-min-pool-size", g_value_get_uint(value), NULL);
+        break;
+    case PROP_OUTPUTS_MAX_POOL_SIZE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the outputs maximum pool size will not take place!");
+            break;
+        }
+        g_object_set(m_hailorecv, "outputs-max-pool-size", g_value_get_uint(value), NULL);
+        break;
+    case PROP_IS_ACTIVE:
+    {
+        gboolean new_is_active = g_value_get_boolean(value);
+
+        if (m_props.m_scheduling_algorithm.was_changed() && (HAILO_SCHEDULING_ALGORITHM_NONE != m_props.m_scheduling_algorithm.get())) {
+            g_error("scheduling-algorithm different than HAILO_SCHEDULING_ALGORITHM_NONE in combination with 'is-active' is not supported.");
+            break;
+        }
+
+        if (m_has_called_activate) {
+            if (m_props.m_is_active.get() && !new_is_active) {
+                // Setting this to false before deactivating to signal hailosend and hailorecv to stop inferring
+                m_props.m_is_active = false;
+                hailo_status status = deactivate_network_group();
+                if (HAILO_SUCCESS != status) {
+                    g_error("Deactivating network group failed, status = %d", status);
+                    return;
+                }
+            } else if (!m_props.m_is_active.get() && new_is_active) {
+                hailo_status status = m_net_group_handle->activate_network_group();
+                if (HAILO_SUCCESS != status) {
+                    g_error("Failed activating network group, status = %d", status);
+                    break;
+                }
+                m_props.m_is_active = true;
+            } else {
+                g_warning("Trying to change is-active property state from %d to %d", m_props.m_is_active.get(), new_is_active);
+                break;
+            }
+        } else {
+            m_props.m_is_active = new_is_active;
+        }
+        break;
+    }
+    case PROP_SCHEDULING_ALGORITHM:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the scheduling algorithm will not take place!");
+            break;
+        }
+        if (m_props.m_is_active.was_changed() && (g_value_get_enum(value) != HAILO_SCHEDULING_ALGORITHM_NONE)) {
+            g_error("scheduling-algorithm different than HAILO_SCHEDULING_ALGORITHM_NONE in combination with 'is-active' is not supported.");
+            break;
+        }
+        m_props.m_scheduling_algorithm = static_cast<hailo_scheduling_algorithm_t>(g_value_get_enum(value));
+        break;
+    case PROP_SCHEDULER_TIMEOUT_MS:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the scheduling timeout will not take place!");
+            break;
+        }
+        if (m_props.m_is_active.was_changed()) {
+            g_error("scheduler usage (scheduler-timeout-ms) in combination with 'is-active' is not supported.");
+            break;
+        }
+        m_props.m_scheduler_timeout_ms = g_value_get_uint(value);
+        break;
+    case PROP_SCHEDULER_THRESHOLD:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the scheduling threshold will not take place!");
+            break;
+        }
+        if (m_props.m_is_active.was_changed()) {
+            g_error("scheduler usage (scheduler-threshold) in combination with 'is-active' is not supported.");
+            break;
+        }
+        m_props.m_scheduler_threshold = g_value_get_uint(value);
+        break;
+    case PROP_SCHEDULER_PRIORITY:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the scheduling priority will not take place!");
+            break;
+        }
+        if (m_props.m_is_active.was_changed()) {
+            g_error("scheduler usage (scheduler-priority) in combination with 'is-active' is not supported.");
+            break;
+        }
+        m_props.m_scheduler_priority = static_cast<guint8>(g_value_get_uint(value));
+        break;
+    case PROP_MULTI_PROCESS_SERVICE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the multi-process-service property will not take place!");
+            break;
+        }
+        m_props.m_multi_process_service = g_value_get_boolean(value);
+        break;
+    case PROP_INPUT_FORMAT_TYPE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the format type will not take place!");
+            break;
+        }
+        m_props.m_input_format_type = static_cast<hailo_format_type_t>(g_value_get_enum(value));
+        break;
+    case PROP_OUTPUT_FORMAT_TYPE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the format type will not take place!");
+            break;
+        }
+        m_props.m_output_format_type = static_cast<hailo_format_type_t>(g_value_get_enum(value));
+        break;
+    case PROP_NMS_SCORE_THRESHOLD:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the score threshold will not take place!");
+            break;
+        }
+        m_props.m_nms_score_threshold = static_cast<gfloat>(g_value_get_float(value));
+        break;
+    case PROP_NMS_IOU_THRESHOLD:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the IoU threshold will not take place!");
+            break;
+        }
+        m_props.m_nms_iou_threshold = static_cast<gfloat>(g_value_get_float(value));
+        break;
+    case PROP_NMS_MAX_PROPOSALS_PER_CLASS:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the max proposals per class will not take place!");
+            break;
+        }
+        m_props.m_nms_max_proposals_per_class = static_cast<guint32>(g_value_get_uint(value));
+        break;
+    default:
+        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
+        break;
+    }
+}
+
+void HailoSyncNetImpl::get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec)
+{
+    GST_DEBUG_OBJECT(m_element, "get_property");
+
+    if ((object == nullptr) || (value == nullptr) || (pspec == nullptr)) {
+        g_error("get_property got null parameter!");
+        return;
+    }
+
+    switch (property_id) {
+    case PROP_DEBUG:
+    {
+        gboolean debug;
+        g_object_get(m_hailosend, "debug", &debug, nullptr);
+        g_value_set_boolean(value, debug);
+        break;
+    }
+    case PROP_DEVICE_ID:
+        g_value_set_string(value, m_props.m_device_id.get());
+        break;
+    case PROP_DEVICE_COUNT:
+        g_value_set_uint(value, m_props.m_device_count.get());
+        break;
+    case PROP_VDEVICE_KEY:
+        g_value_set_uint(value, m_props.m_vdevice_key.get());
+        break;
+    case PROP_HEF_PATH:
+        g_value_set_string(value, m_props.m_hef_path.get());
+        break;
+    case PROP_NETWORK_NAME:
+        g_value_set_string(value, m_props.m_network_name.get());
+        break;
+    case PROP_BATCH_SIZE:
+        g_value_set_uint(value, m_props.m_batch_size.get());
+        break;
+    case PROP_OUTPUTS_MIN_POOL_SIZE:
+    {
+        guint outputs_min_pool_size;
+        g_object_get(m_hailorecv, "outputs-min-pool-size", &outputs_min_pool_size, nullptr);
+        g_value_set_uint(value, outputs_min_pool_size);
+        break;
+    }
+    case PROP_OUTPUTS_MAX_POOL_SIZE:
+    {
+        guint outputs_max_pool_size;
+        g_object_get(m_hailorecv, "outputs-max-pool-size", &outputs_max_pool_size, nullptr);
+        g_value_set_uint(value, outputs_max_pool_size);
+        break;
+    }
+    case PROP_IS_ACTIVE:
+        g_value_set_boolean(value, m_props.m_is_active.get());
+        break;
+    case PROP_SCHEDULING_ALGORITHM:
+        g_value_set_enum(value, m_props.m_scheduling_algorithm.get());
+        break;
+    case PROP_SCHEDULER_TIMEOUT_MS:
+        g_value_set_uint(value, m_props.m_scheduler_timeout_ms.get());
+        break;
+    case PROP_SCHEDULER_THRESHOLD:
+        g_value_set_uint(value, m_props.m_scheduler_threshold.get());
+        break;
+    case PROP_SCHEDULER_PRIORITY:
+        g_value_set_uint(value, m_props.m_scheduler_priority.get());
+        break;
+    case PROP_MULTI_PROCESS_SERVICE:
+        g_value_set_boolean(value, m_props.m_multi_process_service.get());
+        break;
+    case PROP_INPUT_FORMAT_TYPE:
+        g_value_set_enum(value, m_props.m_input_format_type.get());
+        break;
+    case PROP_OUTPUT_FORMAT_TYPE:
+        g_value_set_enum(value, m_props.m_output_format_type.get());
+        break;
+    case PROP_NMS_SCORE_THRESHOLD:
+        g_value_set_float(value, m_props.m_nms_score_threshold.get());
+        break;
+    case PROP_NMS_IOU_THRESHOLD:
+        g_value_set_float(value, m_props.m_nms_iou_threshold.get());
+        break;
+    case PROP_NMS_MAX_PROPOSALS_PER_CLASS:
+        g_value_set_uint(value, m_props.m_nms_max_proposals_per_class.get());
+        break;
+    default:
+        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
+        break;
+    }
+}
+
+hailo_status HailoSyncNetImpl::set_hef()
+{
+    m_net_group_handle = make_unique_nothrow<NetworkGroupHandle>(GST_ELEMENT(m_element));
+    GST_CHECK(nullptr != m_net_group_handle, HAILO_OUT_OF_HOST_MEMORY, m_element, RESOURCE, "Failed allocating memory for network handle!");
+
+    hailo_status status = m_net_group_handle->set_hef(m_props.m_device_id.get(), m_props.m_device_count.get(),
+        m_props.m_vdevice_key.get(), m_props.m_scheduling_algorithm.get(), static_cast<bool>(m_props.m_multi_process_service.get()),
+        m_props.m_hef_path.get());
+    if (HAILO_SUCCESS != status) {
+        return status;
+    }
+
+    if (m_props.m_multi_process_service.get()) {
+        GST_CHECK(m_props.m_scheduling_algorithm.get() != HAILO_SCHEDULING_ALGORITHM_NONE,
+            HAILO_INVALID_OPERATION, m_element, RESOURCE, "To use multi-process-service please set scheduling-algorithm.");
+    }
+
+    if (nullptr == m_props.m_network_name.get()) {
+        // TODO: HRT-4957
+        GST_CHECK(m_net_group_handle->hef()->get_network_groups_names().size() == 1, HAILO_INVALID_ARGUMENT, m_element, RESOURCE,
+            "Network group has to be specified when there are more than one network groups in the HEF!");
+        auto network_group_name = m_net_group_handle->hef()->get_network_groups_names()[0];
+
+        auto networks_infos = m_net_group_handle->hef()->get_network_infos(network_group_name.c_str());
+        GST_CHECK_EXPECTED_AS_STATUS(networks_infos, m_element, RESOURCE, "Getting network infos from network group name was failed, status %d", networks_infos.status());
+        GST_CHECK(networks_infos.value().size() == 1, HAILO_INVALID_ARGUMENT, m_element, RESOURCE,
+            "Network has to be specified when there are more than one network in the network group!");
+
+        std::string default_ng_name = HailoRTDefaults::get_network_name(network_group_name);
+        m_props.m_network_name = g_strdup(default_ng_name.c_str());
+    }
+
+    auto input_vstream_infos = m_net_group_handle->hef()->get_input_vstream_infos(m_props.m_network_name.get());
+    GST_CHECK_EXPECTED_AS_STATUS(input_vstream_infos, m_element, RESOURCE, "Getting input vstream infos from HEF has failed, status = %d",
+        input_vstream_infos.status());
+
+    // TODO: HRT-4095
+    GST_CHECK(1 == input_vstream_infos->size(), HAILO_INVALID_OPERATION, m_element, RESOURCE, "sync hailonet element supports only HEFs with one input for now!");
+
+    auto input_vstream_info = input_vstream_infos.value()[0];
+    GST_HAILOSEND(m_hailosend)->impl->set_input_vstream_infos(input_vstream_infos.release());
+    GST_HAILOSEND(m_hailosend)->impl->set_batch_size(m_props.m_batch_size.get());
+
+    GstBufferPool *pool = gst_buffer_pool_new();
+    GstStructure *config = gst_buffer_pool_get_config(pool);
+
+    auto frame_size = HailoRTCommon::get_frame_size(input_vstream_info, input_vstream_info.format);
+    gst_buffer_pool_config_set_params(config, nullptr, frame_size, 1, 1);
+
+    gboolean result = gst_buffer_pool_set_config(pool, config);
+    GST_CHECK(result, HAILO_INTERNAL_FAILURE, m_element, RESOURCE, "Could not set config buffer pool");
+
+    result = gst_buffer_pool_set_active(pool, TRUE);
+    GST_CHECK(result, HAILO_INTERNAL_FAILURE, m_element, RESOURCE, "Could not set buffer pool active");
+
+    m_pool = pool;
+
+    return HAILO_SUCCESS;
+}
+
+hailo_status HailoSyncNetImpl::configure_network_group()
+{
+    std::unique_lock<std::mutex> lock(m_mutex);
+    g_object_set(m_queue, "max-size-buffers", MAX_BUFFER_COUNT(m_props.m_batch_size.get()), NULL);
+
+    auto network_group_name = get_network_group_name(m_props.m_network_name.get());
+    GST_CHECK_EXPECTED_AS_STATUS(network_group_name, m_element, RESOURCE, "Could not get network group name from name %s, status = %d",
+        m_props.m_network_name.get(), network_group_name.status());
+
+    hailo_status status = m_net_group_handle->configure_network_group(network_group_name->c_str(), m_props.m_scheduling_algorithm.get(), m_props.m_batch_size.get());
+    if (HAILO_SUCCESS != status) {
+        return status;
+    }
+    m_was_configured = true;
+
+    if (m_props.m_scheduler_timeout_ms.was_changed()) {
+        status = m_net_group_handle->set_scheduler_timeout(m_props.m_network_name.get(), m_props.m_scheduler_timeout_ms.get());
+        GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting scheduler timeout failed, status = %d", status);
+    }
+    if (m_props.m_scheduler_threshold.was_changed()) {
+        status = m_net_group_handle->set_scheduler_threshold(m_props.m_network_name.get(), m_props.m_scheduler_threshold.get());
+        GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting scheduler threshold failed, status = %d", status);
+    }
+    if (m_props.m_scheduler_priority.was_changed()) {
+        status = m_net_group_handle->set_scheduler_priority(m_props.m_network_name.get(), m_props.m_scheduler_priority.get());
+        GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting scheduler priority failed, status = %d", status);
+    }
+
+    auto vstreams = m_net_group_handle->create_vstreams(m_props.m_network_name.get(), m_props.m_scheduling_algorithm.get(), m_output_formats,
+        m_props.m_input_format_type.get(), m_props.m_output_format_type.get());
+    GST_CHECK_EXPECTED_AS_STATUS(vstreams, m_element, RESOURCE, "Creating vstreams failed, status = %d", status);
+
+    GST_HAILOSEND(m_hailosend)->impl->set_input_vstreams(std::move(vstreams->first));
+
+    // Check that if one of the NMS params are changed, we have NMS outputs in the model
+    auto has_nms_output = std::any_of(vstreams->second.begin(), vstreams->second.end(), [](const auto &vs)
+    {
+        return HailoRTCommon::is_nms(vs.get_info());
+    });
+
+    for (auto &out_vs : vstreams->second) {
+        if (m_props.m_nms_score_threshold.was_changed()) {
+            GST_CHECK(has_nms_output, HAILO_INVALID_OPERATION, m_element, RESOURCE, "NMS score threshold is set, but there is no NMS output in this model.");
+            if (HailoRTCommon::is_nms(out_vs.get_info())) {
+                status = out_vs.set_nms_score_threshold(m_props.m_nms_score_threshold.get());
+                GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting NMS score threshold failed, status = %d", status);
+            }
+        }
+        if (m_props.m_nms_iou_threshold.was_changed()) {
+            GST_CHECK(has_nms_output, HAILO_INVALID_OPERATION, m_element, RESOURCE, "NMS IoU threshold is set, but there is no NMS output in this model.");
+            if (HailoRTCommon::is_nms(out_vs.get_info())) {
+                status = out_vs.set_nms_iou_threshold(m_props.m_nms_iou_threshold.get());
+                GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting NMS IoU threshold failed, status = %d", status);
+            }
+        }
+        if (m_props.m_nms_max_proposals_per_class.was_changed()) {
+            GST_CHECK(has_nms_output, HAILO_INVALID_OPERATION, m_element, RESOURCE, "NMS max proposals per class is set, but there is no NMS output in this model.");
+            if (HailoRTCommon::is_nms(out_vs.get_info())) {
+                status = out_vs.set_nms_max_proposals_per_class(m_props.m_nms_max_proposals_per_class.get());
+                GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting NMS max proposals per class failed, status = %d", status);
+            }
+        }
+    }
+
+    status = GST_HAILORECV(m_hailorecv)->impl->set_output_vstreams(std::move(vstreams->second), m_props.m_batch_size.get());
+    GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting output vstreams failed, status = %d", status);
+
+    return HAILO_SUCCESS;
+}
+
+hailo_status HailoSyncNetImpl::activate_hailonet()
+{
+    if (HAILO_SCHEDULING_ALGORITHM_NONE != m_props.m_scheduling_algorithm.get()) {
+        m_props.m_is_active = true;
+        return HAILO_SUCCESS;
+    }
+
+    if ((1 == m_sync_hailonet_count) && (!m_props.m_is_active.was_changed())) {
+        m_props.m_is_active = true;
+    }
+
+    if (m_props.m_is_active.get()) {
+        std::unique_lock<std::mutex> lock(m_mutex);
+        hailo_status status = m_net_group_handle->activate_network_group();
+        if (HAILO_SUCCESS != status) {
+            return status;
+        }
+    }
+
+    m_has_called_activate = true;
+
+    return HAILO_SUCCESS;
+}
+
+Expected<std::string> HailoSyncNetImpl::get_network_group_name(const std::string &network_name)
+{
+    for (const auto &network_group_name : m_net_group_handle->hef()->get_network_groups_names()) {
+        // Look for network_group with the given name
+        if ((network_name == network_group_name) || (network_name == HailoRTDefaults::get_network_name(network_group_name))) {
+            return std::string(network_group_name);
+        }
+
+        auto network_infos = m_net_group_handle->hef()->get_network_infos(network_group_name);
+        GST_CHECK_EXPECTED(network_infos, m_element, RESOURCE, "Could not get network infos of group %s, status = %d", network_group_name.c_str(),
+            network_infos.status());
+
+        // Look for network with the given name
+        for (const auto &network_info : network_infos.value()) {
+            if (network_name == network_info.name) {
+                return std::string(network_group_name);
+            }
+        }
+    }
+
+    GST_ELEMENT_ERROR(m_element, RESOURCE, FAILED, ("Failed to get network group name from the name %s!", network_name.c_str()), (NULL));
+    return make_unexpected(HAILO_NOT_FOUND);
+}
+
+hailo_status HailoSyncNetImpl::link_elements()
+{
+    /* Link elements here because only here we have the HEF and the Caps format */
+    if (!gst_element_link_many(m_hailosend, m_queue, m_hailorecv, NULL)) {
+        GST_ELEMENT_ERROR(m_element, RESOURCE, FAILED, ("Could not add link elements in bin!"), (NULL));
+        return HAILO_INTERNAL_FAILURE;
+    }
+
+    return HAILO_SUCCESS;
+}
+
+hailo_status HailoSyncNetImpl::abort_streams()
+{
+    if (!m_props.m_is_active.get()) {
+        return HAILO_SUCCESS;
+    }
+
+    auto status = GST_HAILOSEND(m_hailosend)->impl->abort_vstreams();
+    GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Failed aborting input VStreams of hailosend, status = %d", status);
+    status = GST_HAILORECV(m_hailorecv)->impl->abort_vstreams();
+    GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Failed aborting output VStreams of hailorecv, status = %d", status);
+    return HAILO_SUCCESS;
+}
+
+hailo_status HailoSyncNetImpl::deactivate_network_group()
+{
+    auto was_deactivated = m_net_group_handle->remove_network_group();
+    GST_CHECK_EXPECTED_AS_STATUS(was_deactivated, m_element, RESOURCE, "Failed removing network, status = %d", was_deactivated.status());
+
+    if (was_deactivated.value()) {
+        return clear_vstreams();
+    }
+    return HAILO_SUCCESS;
+}
+
+hailo_status HailoSyncNetImpl::clear_vstreams()
+{
+    if (nullptr != GST_HAILOSEND(m_hailosend)->impl) {
+        hailo_status status = GST_HAILOSEND(m_hailosend)->impl->clear_vstreams();
+        GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Failed clearing input VStreams of hailosend, status = %d", status);
+    }
+
+    if (nullptr != GST_HAILORECV(m_hailorecv)->impl) {
+        hailo_status status = GST_HAILORECV(m_hailorecv)->impl->clear_vstreams();
+        GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Failed clearing output VStreams of hailorecv, status = %d", status);
+    }
+
+    return HAILO_SUCCESS;
+}
+
+gboolean HailoSyncNetImpl::src_pad_event(GstEvent *event)
+{
+    assert(nullptr != event);
+
+    auto parsed_event = HailoSetOutputFormatEvent::parse(event);
+    if (HAILO_SUCCESS != parsed_event.status()) {
+        return FALSE;
+    }
+
+    m_output_formats = std::move(parsed_event->formats);
+    return TRUE;
+}
+
+GstPadProbeReturn HailoSyncNetImpl::sink_probe()
+{
+    hailo_status status = activate_hailonet();
+    GST_CHECK(HAILO_SUCCESS == status, GST_PAD_PROBE_REMOVE, m_element, RESOURCE, "Failed activating network, status = %d", status);
+    return GST_PAD_PROBE_REMOVE;
+}
+
+gboolean HailoSyncNetImpl::is_active()
+{
+    return m_props.m_is_active.get();
+}
+
+hailo_status HailoSyncNetImpl::flush()
+{
+    GstBuffer *buffer = nullptr;
+    GstFlowReturn flow_result = gst_buffer_pool_acquire_buffer(m_pool, &buffer, nullptr);
+    GST_CHECK(GST_FLOW_OK == flow_result, HAILO_INTERNAL_FAILURE, m_element, RESOURCE, "Acquire buffer failed!");
+
+    GstHailoBufferFlagMeta *buffer_meta = GST_HAILO_BUFFER_FLAG_META_ADD(buffer);
+    buffer_meta->flag = BUFFER_FLAG_FLUSH;
+    GST_BUFFER_TIMESTAMP(buffer) = GST_HAILOSEND(m_hailosend)->impl->last_frame_pts();
+
+    GstPad *pad = gst_element_get_static_pad(m_hailosend, "src");
+    flow_result = gst_pad_push(pad, buffer);
+    GST_CHECK(GST_FLOW_OK == flow_result, HAILO_INTERNAL_FAILURE, m_element, RESOURCE, "Pushing buffer to queue has failed!");
+
+    hailo_status status = m_was_flushed_event->wait(WAIT_FOR_FLUSH_TIMEOUT_MS);
+    GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Failed waiting for flushed event, status = %d", status);
+
+    status = m_was_flushed_event->reset();
+    GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Failed resetting flushed event, status = %d", status);
+
+    return HAILO_SUCCESS;
+}
+
+hailo_status HailoSyncNetImpl::signal_was_flushed_event()
+{
+    return m_was_flushed_event->signal();
+}
+
+static void gst_sync_hailonet_init(GstSyncHailoNet *self)
+{
+    if (!do_versions_match(GST_ELEMENT(self))) {
+        return;
+    }
+
+    auto sync_hailonet_impl = HailoSyncNetImpl::create(self);
+    if (!sync_hailonet_impl) {
+        GST_ELEMENT_ERROR(self, RESOURCE, FAILED, ("Creating sync hailonet implementation has failed! status = %d", sync_hailonet_impl.status()), (NULL));
+        return;
+    }
+
+    self->impl = sync_hailonet_impl.release();
+}
+
+static void gst_sync_hailonet_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec)
+{
+    GST_SYNC_HAILONET(object)->impl->set_property(object, property_id, value, pspec);
+}
+
+static void gst_sync_hailonet_get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec)
+{
+    GST_SYNC_HAILONET(object)->impl->get_property(object, property_id, value, pspec);
+}
+
+static gboolean gst_hailorecv_src_pad_event(GstPad */*pad*/, GstObject *parent, GstEvent *event)
+{
+    gboolean result = GST_SYNC_HAILONET(GST_ELEMENT_PARENT(parent))->impl->src_pad_event(event);
+    if (result) {
+        return TRUE;
+    }
+
+    GstBaseTransform *trans = GST_BASE_TRANSFORM_CAST(parent);
+    return GST_BASE_TRANSFORM_GET_CLASS(trans)->src_event(trans, event);
+}
+
+static GstPadProbeReturn gst_sync_hailonet_sink_probe(GstPad *pad, GstPadProbeInfo */*info*/, gpointer /*user_data*/)
+{
+    return GST_SYNC_HAILONET(GST_ELEMENT_PARENT(gst_pad_get_parent(pad)))->impl->sink_probe();
+}
+
+static GstStateChangeReturn gst_sync_hailonet_change_state(GstElement *element, GstStateChange transition)
+{
+    GstStateChangeReturn ret = GST_ELEMENT_CLASS(gst_sync_hailonet_parent_class)->change_state(element, transition);
+    if (GST_STATE_CHANGE_FAILURE == ret) {
+        return ret;
+    }
+
+    auto &sync_hailonet = GST_SYNC_HAILONET(element)->impl;
+    switch (transition) {
+    case GST_STATE_CHANGE_NULL_TO_READY:
+    {
+        hailo_status status = sync_hailonet->link_elements();
+        GST_CHECK(HAILO_SUCCESS == status, GST_STATE_CHANGE_FAILURE, element, RESOURCE, "Linking elements has failed, status = %d\n", status);
+        break;
+    }
+    case GST_STATE_CHANGE_READY_TO_PAUSED:
+    {
+        hailo_status status = sync_hailonet->configure_network_group();
+        GST_CHECK(HAILO_SUCCESS == status, GST_STATE_CHANGE_FAILURE, element, RESOURCE, "Configuring network group failed, status = %d\n", status);
+        break;
+    }
+    case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
+    {
+        hailo_status status = sync_hailonet->abort_streams();
+        GST_CHECK(HAILO_SUCCESS == status, GST_STATE_CHANGE_FAILURE, element, RESOURCE, "Aborting streams has failed, status = %d\n", status);
+        break;
+    }
+    case GST_STATE_CHANGE_READY_TO_NULL:
+    {
+        if (HAILO_SCHEDULING_ALGORITHM_NONE == sync_hailonet->get_props().m_scheduling_algorithm.get()) {
+            auto status = sync_hailonet->deactivate_network_group();
+            GST_CHECK(HAILO_SUCCESS == status, GST_STATE_CHANGE_FAILURE, element, RESOURCE, "Deactivating network group failed, status = %d\n", status);
+        }
+
+        // Cleanup all of hailonet memory
+        sync_hailonet.reset();
+        break;
+    }
+    default:
+        break;
+    }
+
+    return ret;
+}
+
+static void gst_sync_hailonet_flush_callback(GstSyncHailoNet *sync_hailonet, gpointer /*data*/)
+{
+    (void)sync_hailonet->impl->flush();
+}
+
+static void gst_sync_hailonet_inner_queue_overrun_callback(GstElement *queue, gpointer /*udata*/)
+{
+    if (GST_SYNC_HAILONET(GST_ELEMENT_PARENT(queue))->impl->is_active()) {
+        GST_INFO("Inner queue of %s is overrun!", GST_ELEMENT_NAME(GST_ELEMENT_PARENT(queue)));
+    }
+}
+
+static void gst_sync_hailonet_inner_queue_underrun_callback(GstElement *queue, gpointer /*udata*/)
+{
+    if (GST_SYNC_HAILONET(GST_ELEMENT_PARENT(queue))->impl->is_active()) {
+        GST_INFO("Inner queue of %s is underrun!", GST_ELEMENT_NAME(GST_ELEMENT_PARENT(queue)));
+    }
+}

hailort/libhailort/bindings/gstreamer/gst-hailo/sync_gsthailonet.hpp

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the LGPL 2.1 license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#ifndef _GST_SYNC_HAILONET_HPP_
+#define _GST_SYNC_HAILONET_HPP_
+
+#include "common.hpp"
+#include "network_group_handle.hpp"
+#include "hailo/expected.hpp"
+#include "hailo/event.hpp"
+
+#include <atomic>
+#include <condition_variable>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_SYNC_HAILONET (gst_sync_hailonet_get_type())
+#define GST_SYNC_HAILONET(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_SYNC_HAILONET,GstSyncHailoNet))
+#define GST_SYNC_HAILONET_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_SYNC_HAILONET,GstSyncHailoNetClass))
+#define GST_IS_SYNC_HAILONET(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_SYNC_HAILONET))
+#define GST_IS_SYNC_HAILONET_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_SYNC_HAILONET))
+
+class HailoSyncNetImpl;
+struct GstSyncHailoNet
+{
+    GstBin parent;
+    std::unique_ptr<HailoSyncNetImpl> impl;
+};
+
+struct GstSyncHailoNetClass
+{
+    GstBinClass parent;
+};
+
+struct SyncHailoNetProperties final
+{
+public:
+    SyncHailoNetProperties() : m_device_id(nullptr), m_hef_path(nullptr), m_network_name(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE),
+        m_is_active(false), m_device_count(0), m_vdevice_key(DEFAULT_VDEVICE_KEY), m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN),
+        m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS), m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD), m_scheduler_priority(HAILO_SCHEDULER_PRIORITY_NORMAL),
+        m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE), m_input_format_type(HAILO_FORMAT_TYPE_AUTO),
+        m_output_format_type(HAILO_FORMAT_TYPE_AUTO), m_nms_score_threshold(0), m_nms_iou_threshold(0), m_nms_max_proposals_per_class(0)
+
+    {}
+
+    HailoElemProperty<gchar*> m_device_id;
+    HailoElemProperty<gchar*> m_hef_path;
+    HailoElemProperty<gchar*> m_network_name; // This property can be network group name or a network name
+    HailoElemProperty<guint16> m_batch_size;
+    HailoElemProperty<gboolean> m_is_active;
+    HailoElemProperty<guint16> m_device_count;
+    HailoElemProperty<guint32> m_vdevice_key;
+    HailoElemProperty<hailo_scheduling_algorithm_t> m_scheduling_algorithm;
+    HailoElemProperty<guint32> m_scheduler_timeout_ms;
+    HailoElemProperty<guint32> m_scheduler_threshold;
+    HailoElemProperty<guint8> m_scheduler_priority;
+    HailoElemProperty<gboolean> m_multi_process_service;
+    HailoElemProperty<hailo_format_type_t> m_input_format_type;
+    HailoElemProperty<hailo_format_type_t> m_output_format_type;
+    HailoElemProperty<gfloat> m_nms_score_threshold;
+    HailoElemProperty<gfloat> m_nms_iou_threshold;
+    HailoElemProperty<guint32> m_nms_max_proposals_per_class;
+};
+
+class HailoSyncNetImpl final
+{
+public:
+    static Expected<std::unique_ptr<HailoSyncNetImpl>> create(GstSyncHailoNet *element);
+    HailoSyncNetImpl(GstSyncHailoNet *element, GstElement *hailosend, GstElement *queue, GstElement *hailorecv, EventPtr was_flushed_event);
+    ~HailoSyncNetImpl();
+
+    void set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec);
+    void get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec);
+    hailo_status set_hef();
+    hailo_status link_elements();
+    hailo_status configure_network_group();
+    hailo_status activate_hailonet();
+    hailo_status abort_streams();
+
+    gboolean src_pad_event(GstEvent *event);
+    GstPadProbeReturn sink_probe();
+    gboolean is_active();
+    hailo_status flush();
+    hailo_status signal_was_flushed_event();
+
+    hailo_status deactivate_network_group();
+    SyncHailoNetProperties &get_props() {
+        return m_props;
+    }
+
+private:
+    void init_ghost_sink();
+    void init_ghost_src();
+    Expected<std::string> get_network_group_name(const std::string &network_name);
+
+    hailo_status clear_vstreams();
+
+    static std::atomic_uint32_t m_sync_hailonet_count;
+    static std::mutex m_mutex;
+    GstSyncHailoNet *m_element;
+    SyncHailoNetProperties m_props;
+    std::vector<hailo_format_with_name_t> m_output_formats;
+    GstElement *m_hailosend;
+    GstElement *m_queue;
+    GstElement *m_hailorecv;
+    std::unique_ptr<NetworkGroupHandle> m_net_group_handle;
+    bool m_was_configured;
+    bool m_has_called_activate;
+    EventPtr m_was_flushed_event;
+    GstBufferPool *m_pool;
+};
+
+GType gst_sync_hailonet_get_type(void);
+
+G_END_DECLS
+
+#endif /* _GST_SYNC_HAILONET_HPP_ */

hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py

@@ -127,10 +127,8 @@ class ExceptionWrapper(object):
             raise HailoRTInvalidFrameException("An invalid frame was received") from libhailort_exception
         if string_error_code == "HAILO_TIMEOUT":
             raise HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred") from libhailort_exception
-        if string_error_code == "HAILO_STREAM_ABORTED_BY_HW":
+        if string_error_code == "HAILO_STREAM_ABORT":
-            raise HailoRTStreamAborted("Stream aborted due to an external event") from libhailort_exception
+            raise HailoRTStreamAborted("Stream was aborted") from libhailort_exception
-        if string_error_code == "HAILO_STREAM_ABORTED_BY_USER":
-            raise HailoRTStreamAbortedByUser("Stream was aborted by user") from libhailort_exception
 
         if string_error_code == "HAILO_INVALID_OPERATION":
             raise HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information") from libhailort_exception
@@ -720,10 +718,9 @@ class ConfiguredNetwork(object):
             return self._configured_network.get_vstream_names_from_stream_name(stream_name)
 
     def set_scheduler_timeout(self, timeout_ms, network_name=None):
-        """Sets the maximum time period that may pass before getting run time from the scheduler,
+        """Sets the maximum time period that may pass before receiving run time from the scheduler.
-            even without reaching the minimum required send requests (e.g. threshold - see set_scheduler_threshold()),
+            This will occur providing at least one send request has been sent, there is no minimum requirement for send
-            as long as at least one send request has been sent.
+            requests, (e.g. threshold - see set_scheduler_threshold()).
-            This time period is measured since the last time the scheduler gave this network group run time.
 
         Args:
             timeout_ms (int): Timeout in milliseconds.
@@ -1058,10 +1055,25 @@ class InferVStreams(object):
             max_proposals_per_class (int): NMS max proposals per class to set.
 
         Note:
+            This function must be called before starting inference!
             This function will fail in cases where there is no output with NMS operations on the CPU.
         """
         return self._infer_pipeline.set_nms_max_proposals_per_class(max_proposals_per_class)
 
+    def set_nms_max_accumulated_mask_size(self, max_accumulated_mask_size):
+        """Set maximum accumulated mask size for all the detections in a frame.
+            Used in order to change the output buffer frame size,
+            in cases where the output buffer is too small for all the segmentation detections.
+
+        Args:
+            max_accumulated_mask_size (int): NMS max accumulated mask size.
+
+        Note:
+            This function must be called before starting inference!
+            This function will fail in cases where there is no output with NMS operations on the CPU.
+        """
+        return self._infer_pipeline.set_nms_max_accumulated_mask_size(max_accumulated_mask_size)
+
     def __exit__(self, *args):
         self._infer_pipeline.release()
         return False
@@ -1487,8 +1499,8 @@ class HailoFormatFlags(_pyhailort.FormatFlags):
 
 SUPPORTED_PROTOCOL_VERSION = 2
 SUPPORTED_FW_MAJOR = 4
-SUPPORTED_FW_MINOR = 16
+SUPPORTED_FW_MINOR = 17
-SUPPORTED_FW_REVISION = 2
+SUPPORTED_FW_REVISION = 0
 
 MEGA_MULTIPLIER = 1000.0 * 1000.0
 
@@ -3120,6 +3132,20 @@ class OutputVStream(object):
         """
         return self._recv_object.set_nms_max_proposals_per_class(max_proposals_per_class)
 
+    def set_nms_max_accumulated_mask_size(self, max_accumulated_mask_size):
+        """Set maximum accumulated mask size for all the detections in a frame.
+            Used in order to change the output buffer frame size,
+            in cases where the output buffer is too small for all the segmentation detections.
+
+        Args:
+            max_accumulated_mask_size (int): NMS max accumulated mask size.
+
+        Note:
+            This function must be called before starting inference!
+            This function will fail in cases where there is no output with NMS operations on the CPU.
+        """
+        return self._recv_object.set_nms_max_accumulated_mask_size(max_accumulated_mask_size)
+
 
 class OutputVStreams(object):
     """Output virtual streams pipelines that allows to receive data, to be used as a context manager."""

hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/main.py

@@ -23,7 +23,7 @@ class PlatformCommands:
         'fw-update': ('Firmware update tool', FWUpdaterCLI),
         'ssb-update': ('Second stage boot update tool', SSBUpdaterCLI),
         'fw-config': ('Firmware configuration tool', FWConfigCommandCLI),
-        'udp-rate-limiter': ('Limit UDP rate', UDPRateLimiterCLI),
+        'udp-rate-limiter': ('Limit the UDP rate', UDPRateLimiterCLI),
         'fw-control': ('Useful firmware control operations', ControlCommandCLI),
         'fw-logger': ('Download fw logs to a file', LoggerCommandCLI),
         'scan': ('Scans for devices (Ethernet or PCIE)', ScanCommandCLI),

hailort/libhailort/bindings/python/platform/setup.py

@@ -69,6 +69,6 @@ if __name__ == "__main__":
             "linux_aarch64",
         ],
         url="https://hailo.ai/",
-        version="4.16.2",
+        version="4.17.0",
         zip_safe=False,
     )

hailort/libhailort/bindings/python/src/CMakeLists.txt

@@ -49,7 +49,7 @@ set_target_properties(_pyhailort PROPERTIES
     # VISIBILITY_INLINES_HIDDEN YES
 )
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 target_link_libraries(_pyhailort PRIVATE HailoRT::libhailort)
 if(WIN32)

hailort/libhailort/bindings/python/src/bindings_common.hpp

@@ -48,7 +48,7 @@ public:
         case HAILO_FORMAT_ORDER_HAILO_NMS:
             return { HailoRTCommon::get_nms_host_shape_size(vstream_info.nms_shape) };
         case HAILO_FORMAT_ORDER_HAILO_NMS_WITH_BYTE_MASK: {
-            return { HailoRTCommon::get_nms_with_byte_mask_host_shape_size(vstream_info.nms_shape, user_format) };
+            return {HailoRTCommon::get_nms_host_frame_size(vstream_info.nms_shape, user_format) / HailoRTCommon::get_format_data_bytes(user_format)};
         }
         case HAILO_FORMAT_ORDER_NC:
             return {shape.features};

hailort/libhailort/bindings/python/src/internal/CMakeLists.txt

@@ -1,44 +0,0 @@
-cmake_minimum_required(VERSION 3.15.0)
-
-include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/spdlog.cmake)
-include(${HAILO_EXTERNALS_CMAKE_SCRIPTS}/readerwriterqueue.cmake)
-
-pybind11_add_module(_pyhailort_internal SHARED
-    pyhailort_internal.cpp
-    control_api.cpp
-    $<TARGET_OBJECTS:libhailort>
-)
-
-add_dependencies(_pyhailort_internal libhailort)
-
-set_target_properties(_pyhailort_internal PROPERTIES
-    CXX_STANDARD              14
-    CXX_STANDARD_REQUIRED     YES
-)
-
-target_include_directories(_pyhailort_internal
-    PRIVATE
-    $<BUILD_INTERFACE:${PYHAILORT_DIR}>
-    $<BUILD_INTERFACE:${HAILORT_INC_DIR}>
-    $<BUILD_INTERFACE:${HAILORT_COMMON_DIR}>
-    $<BUILD_INTERFACE:${HAILORT_SRC_DIR}>
-    $<BUILD_INTERFACE:${COMMON_INC_DIR}>
-    $<BUILD_INTERFACE:${DRIVER_INC_DIR}>
-)
-
-target_link_libraries(_pyhailort_internal PRIVATE
-    hef_proto
-    spdlog::spdlog
-    readerwriterqueue
-    profiler_proto
-    scheduler_mon_proto)
-if(HAILO_BUILD_SERVICE)
-    target_link_libraries(_pyhailort_internal PRIVATE grpc++_unsecure hailort_rpc_grpc_proto)
-endif()
-
-if(WIN32)
-    target_link_libraries(_pyhailort_internal PRIVATE Ws2_32 Iphlpapi Shlwapi)
-endif()
-
-target_compile_options(_pyhailort_internal PRIVATE ${HAILORT_COMPILE_OPTIONS})
-exclude_archive_libs_symbols(_pyhailort_internal)

hailort/libhailort/bindings/python/src/internal/control_api.cpp

@@ -1,256 +0,0 @@
-/**
- * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
- * Distributed under the MIT license (https://opensource.org/licenses/MIT)
- **/
-/**
- * @file control_api.cpp
- * @brief Defines binding to control functions
- *
- **/
-
-#include "control_api.hpp"
-#include "utils.hpp"
-#include "hailo/device.hpp"
-#include "common/utils.hpp"
-
-namespace hailort
-{
-
-void ControlWrapper::set_clock_freq(DeviceWrapper &device, uint32_t clock_freq)
-{
-    auto status = Control::set_clock_freq(*device, clock_freq);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::close_all_streams(DeviceWrapper &device)
-{
-    auto status = Control::close_all_streams(*device);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::config_ahb_to_axi(DeviceWrapper &device, bool use_64bit_data_only)
-{
-    CONTROL_PROTOCOL__config_core_top_type_t config_type = CONTROL_PROTOCOL__CONFIG_CORE_TOP_TYPE_AHB_TO_AXI;
-    CONTROL_PROTOCOL__config_core_top_params_t params = {0};
-    params.ahb_to_axi.enable_use_64bit_data_only = use_64bit_data_only;
-
-    auto status = Control::config_core_top(*device, config_type, &params);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::phy_operation(DeviceWrapper &device, CONTROL_PROTOCOL__phy_operation_t operation_type)
-{
-    auto status = Control::phy_operation(*device, operation_type);
-    VALIDATE_STATUS(status);
-}
-
-uint32_t ControlWrapper::latency_measurement_read(DeviceWrapper &device)
-{
-    uint32_t inbound_to_outbound_latency_nsec = 0;
-
-    auto status = Control::latency_measurement_read(*device, &inbound_to_outbound_latency_nsec);
-    VALIDATE_STATUS(status);
-
-    return inbound_to_outbound_latency_nsec;
-}
-
-void ControlWrapper::latency_measurement_config(DeviceWrapper &device, uint8_t latency_measurement_en,
-    uint32_t inbound_start_buffer_number, uint32_t outbound_stop_buffer_number, uint32_t inbound_stream_index,
-    uint32_t outbound_stream_index)
-{
-    auto status = Control::latency_measurement_config(*device, latency_measurement_en, inbound_start_buffer_number,
-            outbound_stop_buffer_number, inbound_stream_index, outbound_stream_index);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::start_firmware_update(DeviceWrapper &device)
-{
-    auto status = Control::start_firmware_update(*device);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::finish_firmware_update(DeviceWrapper &device)
-{
-    auto status = Control::finish_firmware_update(*device);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::write_firmware_update(DeviceWrapper &device, uint32_t offset, py::bytes data, uint32_t length)
-{
-    auto status = Control::write_firmware_update(*device, offset, (uint8_t*)std::string(data).c_str(), length);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::validate_firmware_update(DeviceWrapper &device, py::bytes md5_raw_data, uint32_t firmware_size)
-{
-    MD5_SUM_t expected_md5 = {0};
-    memcpy(&expected_md5, (uint8_t*)std::string(md5_raw_data).c_str(), sizeof(expected_md5));
-
-    auto status = Control::validate_firmware_update(*device, &expected_md5, firmware_size);
-    VALIDATE_STATUS(status);
-}
-
-py::bytes ControlWrapper::sensor_get_config(DeviceWrapper &device, uint32_t section_index, uint32_t offset, uint32_t data_length)
-{
-    std::unique_ptr<std::string> response = make_unique_nothrow<std::string>(data_length, '\x00');
-    VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY);
-    
-    auto status = Control::sensor_get_config(*device, section_index, offset, data_length, (uint8_t*)(response->data()));
-    VALIDATE_STATUS(status);
-
-     return *response;
-}
-
-void ControlWrapper::idle_time_set_measurement(DeviceWrapper &device, bool measurement_enable)
-{
-    auto status = Control::idle_time_set_measurement(*device, measurement_enable);
-    VALIDATE_STATUS(status);
-}
-
-uint64_t ControlWrapper::idle_time_get_measurement(DeviceWrapper &device)
-{
-    uint64_t measurement = 0;
-
-    auto status = Control::idle_time_get_measurement(*device, &measurement);
-    VALIDATE_STATUS(status);
-
-    return measurement;
-}
-
-void ControlWrapper::d2h_notification_manager_set_host_info(DeviceWrapper &device, uint16_t host_port, uint32_t host_ip_address)
-{
-    auto status = Control::d2h_notification_manager_set_host_info(*device, host_port, host_ip_address);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::d2h_notification_manager_send_host_info_notification(DeviceWrapper &device, uint8_t notification_priority)
-{
-    auto status = Control::d2h_notification_manager_send_host_info_notification(*device, notification_priority);
-    VALIDATE_STATUS(status);
-}
-
-/* Context switch */
-void ControlWrapper::set_context_switch_breakpoint(DeviceWrapper &device, 
-        uint8_t breakpoint_id,
-        bool break_at_any_network_group_index, uint8_t network_group_index, 
-        bool break_at_any_batch_index, uint16_t batch_index, 
-        bool break_at_any_context_index,uint8_t context_index, 
-        bool break_at_any_action_index, uint16_t action_index) 
-{
-    CONTROL_PROTOCOL__context_switch_breakpoint_control_t breakpoint_control = 
-        CONTROL_PROTOCOL__CONTEXT_SWITCH_BREAKPOINT_CONTROL_SET;
-    CONTROL_PROTOCOL__context_switch_breakpoint_data_t breakpoint_data = {
-        break_at_any_network_group_index,
-        network_group_index,
-        break_at_any_batch_index,
-        batch_index,
-        break_at_any_context_index,
-        context_index,
-        break_at_any_action_index,
-        action_index};
-
-    auto status = Control::config_context_switch_breakpoint(*device, breakpoint_id, breakpoint_control, &breakpoint_data);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::continue_context_switch_breakpoint(DeviceWrapper &device, uint8_t breakpoint_id) 
-{
-    CONTROL_PROTOCOL__context_switch_breakpoint_control_t breakpoint_control = 
-        CONTROL_PROTOCOL__CONTEXT_SWITCH_BREAKPOINT_CONTROL_CONTINUE;
-    CONTROL_PROTOCOL__context_switch_breakpoint_data_t breakpoint_data = {false,0,false,0,false,0,false,0};
-
-    auto status = Control::config_context_switch_breakpoint(*device, breakpoint_id, 
-            breakpoint_control, &breakpoint_data);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::clear_context_switch_breakpoint(DeviceWrapper &device, uint8_t breakpoint_id) 
-{
-    CONTROL_PROTOCOL__context_switch_breakpoint_control_t breakpoint_control = 
-        CONTROL_PROTOCOL__CONTEXT_SWITCH_BREAKPOINT_CONTROL_CLEAR;
-    CONTROL_PROTOCOL__context_switch_breakpoint_data_t breakpoint_data = {false,0,false,0,false,0,false,0};
-
-    auto status = Control::config_context_switch_breakpoint(*device, breakpoint_id,
-            breakpoint_control, &breakpoint_data);
-    VALIDATE_STATUS(status);
-}
-
-uint8_t ControlWrapper::get_context_switch_breakpoint_status(DeviceWrapper &device, uint8_t breakpoint_id) 
-{
-    CONTROL_PROTOCOL__context_switch_debug_sys_status_t breakpoint_status = 
-        CONTROL_PROTOCOL__CONTEXT_SWITCH_DEBUG_SYS_STATUS_COUNT;
-
-    auto status = Control::get_context_switch_breakpoint_status(*device, breakpoint_id,
-            &breakpoint_status);
-    VALIDATE_STATUS(status);
-
-    return static_cast<uint8_t>(breakpoint_status);
-}
-
-void ControlWrapper::config_context_switch_timestamp(DeviceWrapper &device, uint16_t batch_index) 
-{
-    auto status = Control::config_context_switch_timestamp(*device, batch_index, true);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::remove_context_switch_timestamp_configuration(DeviceWrapper &device) 
-{
-    auto status = Control::config_context_switch_timestamp(*device, 0, false);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::enable_debugging(DeviceWrapper &device, bool is_rma)
-{
-    auto status = Control::enable_debugging(*device, is_rma);
-    VALIDATE_STATUS(status);
-}
-
-void ControlWrapper::add_to_python_module(py::module &m)
-{
-    m.def("_set_clock_freq", &ControlWrapper::set_clock_freq);
-    m.def("close_all_streams", &ControlWrapper::close_all_streams);
-    m.def("config_ahb_to_axi", &ControlWrapper::config_ahb_to_axi);
-    m.def("phy_operation", &ControlWrapper::phy_operation);
-    m.def("latency_measurement_read", &ControlWrapper::latency_measurement_read);
-    m.def("latency_measurement_config", &ControlWrapper::latency_measurement_config);
-    m.def("start_firmware_update", &ControlWrapper::start_firmware_update);
-    m.def("finish_firmware_update", &ControlWrapper::finish_firmware_update);
-    m.def("write_firmware_update", &ControlWrapper::write_firmware_update);
-    m.def("validate_firmware_update", &ControlWrapper::validate_firmware_update);
-    m.def("sensor_get_config", &ControlWrapper::sensor_get_config);
-    m.def("idle_time_set_measurement", &ControlWrapper::idle_time_set_measurement);
-    m.def("idle_time_get_measurement", &ControlWrapper::idle_time_get_measurement);
-    m.def("d2h_notification_manager_set_host_info", &ControlWrapper::d2h_notification_manager_set_host_info);
-    m.def("d2h_notification_manager_send_host_info_notification", &ControlWrapper::d2h_notification_manager_send_host_info_notification);
-    m.def("set_context_switch_breakpoint", &set_context_switch_breakpoint);
-    m.def("continue_context_switch_breakpoint", &continue_context_switch_breakpoint);
-    m.def("clear_context_switch_breakpoint", &clear_context_switch_breakpoint);
-    m.def("get_context_switch_breakpoint_status", &get_context_switch_breakpoint_status);
-    m.def("config_context_switch_timestamp", &config_context_switch_timestamp);
-    m.def("remove_context_switch_timestamp_configuration", &remove_context_switch_timestamp_configuration);
-    m.def("enable_debugging", &enable_debugging);
-    
-    // TODO: HRT-5764 - Remove 'py::module_local()' when removing _pyhailort_internal from external
-    // py::module_local() is needed because these enums are currently in both _pyhailort and _pyhailort_internal, 
-    // and when trying to import one of them on the python side you will get the error:
-    // ImportError: generic_type: type "enum_name" is already registered!
-    // py::module_local() tells pybind11 to keep the external class/enum binding localized to the module. 
-    py::enum_<CONTROL_PROTOCOL__context_switch_debug_sys_status_t>(m, "ContextSwitchBreakpointStatus", py::module_local())
-        .value("CONTEXT_SWITCH_BREAKPOINT_STATUS_CLEARED",CONTROL_PROTOCOL__CONTEXT_SWITCH_DEBUG_SYS_STATUS_CLEARED)
-        .value("CONTEXT_SWITCH_BREAKPOINT_STATUS_WAITING_FOR_BREAKPOINT",CONTROL_PROTOCOL__CONTEXT_SWITCH_DEBUG_SYS_STATUS_WAITING_FOR_BREAKPOINT)
-        .value("CONTEXT_SWITCH_BREAKPOINT_STATUS_REACHED_BREAKPOINT",CONTROL_PROTOCOL__CONTEXT_SWITCH_DEBUG_SYS_STATUS_REACHED_BREAKPOINT)
-        ;
-
-    py::enum_<CONTROL_PROTOCOL__phy_operation_t>(m, "CONTROL_PROTOCOL__phy_operation_t", py::module_local())
-        .value("PHY_OPERATION_RESET", CONTROL_PROTOCOL__PHY_OPERATION_RESET)
-        ;
-
-    py::enum_<CONTROL_PROTOCOL__mipi_deskew_enable_t>(m, "CONTROL_PROTOCOL__mipi_deskew_enable_t", py::module_local())
-        .value("MIPI__DESKEW_FORCE_DISABLE", CONTROL_PROTOCOL__MIPI_DESKEW__FORCE_DISABLE)
-        .value("MIPI__DESKEW_FORCE_ENABLE", CONTROL_PROTOCOL__MIPI_DESKEW__FORCE_ENABLE)
-        .value("MIPI__DESKEW_DEFAULT", CONTROL_PROTOCOL__MIPI_DESKEW__DEFAULT)
-        ;
-
-}
-
-} /* namespace hailort */

hailort/libhailort/bindings/python/src/internal/control_api.hpp

@@ -1,68 +0,0 @@
-/**
- * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
- * Distributed under the MIT license (https://opensource.org/licenses/MIT)
- **/
-/**
- * @file control_api.hpp
- * @brief Defines binding to control functions
- *
- **/
-
-#ifndef _CONTROL_API_HPP_
-#define _CONTROL_API_HPP_
-
-#include "device_common/control.hpp"
-#include "utils.hpp"
-
-#include "device_api.hpp"
-
-#include <pybind11/pybind11.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/numpy.h>
-#include <pybind11/detail/common.h>
-#include <pybind11/stl.h>
-#include <pybind11/complex.h>
-#include <pybind11/functional.h>
-
-namespace hailort
-{
-
-class ControlWrapper {
-public:
-    static void add_to_python_module(py::module &m);
-
-    static void set_clock_freq(DeviceWrapper &device, uint32_t clock_freq);
-    static void close_all_streams(DeviceWrapper &device);
-    static void config_ahb_to_axi(DeviceWrapper &device, bool use_64bit_data_only);
-    static void phy_operation(DeviceWrapper &device, CONTROL_PROTOCOL__phy_operation_t operation_type);
-    static uint32_t latency_measurement_read(DeviceWrapper &device);
-    static void latency_measurement_config(DeviceWrapper &device, uint8_t latency_measurement_en,
-        uint32_t inbound_start_buffer_number, uint32_t outbound_stop_buffer_number, uint32_t inbound_stream_index,
-        uint32_t outbound_stream_index);
-    static void start_firmware_update(DeviceWrapper &device);
-    static void finish_firmware_update(DeviceWrapper &device);
-    static void write_firmware_update(DeviceWrapper &device, uint32_t offset, py::bytes data, uint32_t length);
-    static void validate_firmware_update(DeviceWrapper &device, py::bytes md5_raw_data, uint32_t firmware_size);
-    static py::bytes sensor_get_config(DeviceWrapper &device, uint32_t section_index, uint32_t offset, uint32_t data_length);
-    static void idle_time_set_measurement(DeviceWrapper &device, bool measurement_enable);
-    static uint64_t idle_time_get_measurement(DeviceWrapper &device);
-    static void d2h_notification_manager_set_host_info(DeviceWrapper &device, uint16_t host_port, uint32_t host_ip_address);
-    static void d2h_notification_manager_send_host_info_notification(DeviceWrapper &device, uint8_t notification_priority);
-    static void enable_debugging(DeviceWrapper &device, bool is_rma);
-
-    /* Context switch */
-    static void set_context_switch_breakpoint(DeviceWrapper &device, uint8_t breakpoint_id,
-        bool break_at_any_network_group_index, uint8_t network_group_index, 
-        bool break_at_any_batch_index, uint16_t batch_index, 
-        bool break_at_any_context_index,uint8_t context_index, 
-        bool break_at_any_action_index, uint16_t action_index);
-    static void continue_context_switch_breakpoint(DeviceWrapper &device, uint8_t breakpoint_id);
-    static void clear_context_switch_breakpoint(DeviceWrapper &device, uint8_t breakpoint_id);
-    static uint8_t get_context_switch_breakpoint_status(DeviceWrapper &device, uint8_t breakpoint_id);
-    static void config_context_switch_timestamp(DeviceWrapper &device, uint16_t batch_index);
-    static void remove_context_switch_timestamp_configuration(DeviceWrapper &device);
-};
-
-} /* namespace hailort */
-
-#endif /* _CONTROL_API_HPP_ */

hailort/libhailort/bindings/python/src/internal/pyhailort_internal.cpp

@@ -1,405 +0,0 @@
-
-
-#include "hailo/hailort.h"
-
-#include "transform/transform_internal.hpp"
-#include "bindings_common.hpp"
-
-#include "pyhailort_internal.hpp"
-#include "control_api.hpp"
-#include "utils.hpp"
-#include "utils.h"
-
-#include <pybind11/pybind11.h>
-#include <pybind11/numpy.h>
-#include <pybind11/detail/common.h>
-#include <pybind11/stl.h>
-#include <pybind11/complex.h>
-#include <pybind11/functional.h>
-#include <vector>
-
-
-namespace hailort
-{
-// TODO: Remove (HRT-9944)
-// Duplicated for hailo post process test with python API.
-static const uint32_t TEST_NUM_OF_CLASSES = 80;
-
-
-Expected<Buffer> get_expected_buffer_float32()
-{
-    static const uint32_t DETECTION_CLASS_ID_1 = 0;
-    static const float32_t CLASS_ID_1_DETECTION_COUNT = 5;
-    static const uint32_t DETECTION_CLASS_ID_3 = 2;
-    static const float32_t CLASS_ID_3_DETECTION_COUNT = 2;
-    static const uint32_t DETECTION_CLASS_ID_8 = 7;
-    static const float32_t CLASS_ID_8_DETECTION_COUNT = 1;
-    static const uint32_t DETECTION_CLASS_ID_26 = 25;
-    static const float32_t CLASS_ID_26_DETECTION_COUNT = 1;
-
-    static const hailo_bbox_float32_t bbox1_0 = {
-        /*.y_min =*/ 0.5427529811859131f,
-        /*.x_min =*/ 0.2485126256942749f,
-        /*.y_max =*/ 0.6096446067f,
-        /*.x_max =*/ 0.27035075984f,
-        /*.score =*/ 0.7761699557304382f,
-    };
-    
-    static const hailo_bbox_float32_t bbox1_1 = {
-        /*.y_min =*/ 0.5454554557800293f,
-        /*.x_min =*/ 0.33257606625556948f,
-        /*.y_max =*/ 0.7027952075f,
-        /*.x_max =*/ 0.40901548415f,
-        /*.score =*/ 0.7637669444084168f,
-    };
-    
-    static const hailo_bbox_float32_t bbox1_2 = {
-        /*.y_min =*/ 0.5521867275238037f,
-        /*.x_min =*/ 0.19988654553890229f,
-        /*.y_max =*/ 0.60256312787f,
-        /*.x_max =*/ 0.21917282976f,
-        /*.score =*/ 0.7451231479644775f,
-    };
-
-    static const hailo_bbox_float32_t bbox1_3 = {
-        /*.y_min =*/ 0.5514537692070007f,
-        /*.x_min =*/ 0.2693796157836914f,
-        /*.y_max =*/ 0.60397491604f,
-        /*.x_max =*/ 0.28537025302f,
-        /*.score =*/ 0.3756354749202728f,
-    };
-    
-    static const hailo_bbox_float32_t bbox1_4 = {
-        /*.y_min =*/ 0.553998589515686f,
-        /*.x_min =*/ 0.18612079322338105f,
-        /*.y_max =*/ 0.58339602686f,
-        /*.x_max =*/ 0.2008818537f,
-        /*.score =*/ 0.3166312277317047f,
-    };
-
-    static const hailo_bbox_float32_t bbox3_0 = {
-        /*.y_min =*/ 0.5026738047599793f,
-        /*.x_min =*/ -0.005611047148704529f,
-        /*.y_max =*/ 0.65071095526f,
-        /*.x_max =*/ 0.13888412714f,
-        /*.score =*/ 0.5734351277351379f,
-    };
-
-    static const hailo_bbox_float32_t bbox3_1 = {
-        /*.y_min =*/ 0.5620155334472656f,
-        /*.x_min =*/ 0.16757474839687348f,
-        /*.y_max =*/ 0.58410947769f,
-        /*.x_max =*/ 0.19325175508f,
-        /*.score =*/ 0.4062519371509552f,
-    };
-
-    static const hailo_bbox_float32_t bbox8_0 = {
-        /*.y_min =*/ 0.5028372406959534f,
-        /*.x_min =*/ -0.0017736181616783143f,
-        /*.y_max =*/ 0.65114967525f,
-        /*.x_max =*/ 0.13592261821f,
-        /*.score =*/ 0.4223918318748474f,
-    };
-
-    static const hailo_bbox_float32_t bbox26_0 = {
-        /*.y_min =*/ 0.5854946374893189f,
-        /*.x_min =*/ 0.2693060040473938f,
-        /*.y_max =*/ 0.68259389698f,
-        /*.x_max =*/ 0.38090330362f,
-        /*.score =*/ 0.6338639259338379f,
-    };
-
-    static const uint32_t DETECTION_COUNT = 9;
-    auto buffer_size = (DETECTION_COUNT * sizeof(hailo_bbox_float32_t)) + (TEST_NUM_OF_CLASSES * sizeof(float32_t));
-    auto buffer_expected = Buffer::create(buffer_size, 0);
-    CHECK_EXPECTED(buffer_expected);
-    auto buffer = buffer_expected.release();
-
-    size_t offset = 0;
-    for (uint32_t class_index = 0; class_index < TEST_NUM_OF_CLASSES; class_index++) {
-        if (DETECTION_CLASS_ID_1 == class_index) {
-            memcpy(buffer.data() + offset, &CLASS_ID_1_DETECTION_COUNT, sizeof(CLASS_ID_1_DETECTION_COUNT));
-            offset += sizeof(CLASS_ID_1_DETECTION_COUNT);
-
-            memcpy(buffer.data() + offset, &bbox1_0, sizeof(bbox1_0));
-            offset += sizeof(bbox1_0);
-
-            memcpy(buffer.data() + offset, &bbox1_1, sizeof(bbox1_1));
-            offset += sizeof(bbox1_1);
-
-            memcpy(buffer.data() + offset, &bbox1_2, sizeof(bbox1_2));
-            offset += sizeof(bbox1_2);
-
-            memcpy(buffer.data() + offset, &bbox1_3, sizeof(bbox1_3));
-            offset += sizeof(bbox1_3);
-
-            memcpy(buffer.data() + offset, &bbox1_4, sizeof(bbox1_4));
-            offset += sizeof(bbox1_4);
-        }
-        else if (DETECTION_CLASS_ID_3 == class_index) {
-            memcpy(buffer.data() + offset, &CLASS_ID_3_DETECTION_COUNT, sizeof(CLASS_ID_3_DETECTION_COUNT));
-            offset += sizeof(CLASS_ID_3_DETECTION_COUNT);
-
-            memcpy(buffer.data() + offset, &bbox3_0, sizeof(bbox3_0));
-            offset += sizeof(bbox3_0);
-
-            memcpy(buffer.data() + offset, &bbox3_1, sizeof(bbox3_1));
-            offset += sizeof(bbox3_1);
-        }
-        else if (DETECTION_CLASS_ID_8 == class_index) {
-            memcpy(buffer.data() + offset, &CLASS_ID_8_DETECTION_COUNT, sizeof(CLASS_ID_8_DETECTION_COUNT));
-            offset += sizeof(CLASS_ID_8_DETECTION_COUNT);
-
-            memcpy(buffer.data() + offset, &bbox8_0, sizeof(bbox8_0));
-            offset += sizeof(bbox8_0);
-        }
-        else if (DETECTION_CLASS_ID_26 == class_index) {
-            memcpy(buffer.data() + offset, &CLASS_ID_26_DETECTION_COUNT, sizeof(CLASS_ID_26_DETECTION_COUNT));
-            offset += sizeof(CLASS_ID_26_DETECTION_COUNT);
-
-            memcpy(buffer.data() + offset, &bbox26_0, sizeof(bbox26_0));
-            offset += sizeof(bbox26_0);
-        }
-        else {
-            offset += sizeof(float32_t);
-        }
-    }
-
-    return buffer;
-}
-
-py::array PyhailortInternal::get_yolov5_post_process_expected_buffer()
-{
-    auto buffer = get_expected_buffer_float32();
-    VALIDATE_EXPECTED(buffer);
-
-    auto type = py::dtype(HailoRTBindingsCommon::convert_format_type_to_string(HAILO_FORMAT_TYPE_FLOAT32));
-    auto shape = *py::array::ShapeContainer({buffer->size()});
-
-    // Note: The ownership of the buffer is transferred to Python wrapped as a py::array.
-    //       When the py::array isn't referenced anymore in Python and is destructed, the py::capsule's dtor
-    //       is called too (and it deletes the raw buffer)
-    auto unmanaged_addr_exp = buffer->storage().release();
-    VALIDATE_EXPECTED(unmanaged_addr_exp);
-    const auto unmanaged_addr = unmanaged_addr_exp.release();
-    return py::array(type, shape, unmanaged_addr,
-        py::capsule(unmanaged_addr, [](void *p) { delete reinterpret_cast<uint8_t*>(p); }));
-}
-
-void PyhailortInternal::demux_output_buffer(
-    py::bytes src, const hailo_format_t &src_format, const hailo_3d_image_shape_t &src_shape,
-    std::map<std::string, py::array> dst_buffers, const LayerInfo &mux_layer_info)
-{
-    const size_t hw_frame_size = HailoRTCommon::get_frame_size(src_shape, src_format);
-    auto expected_output_demuxer = OutputDemuxerBase::create(hw_frame_size, mux_layer_info);
-    VALIDATE_EXPECTED(expected_output_demuxer);
-
-    auto demuxer = expected_output_demuxer.release();
-
-    std::map<std::string, MemoryView> dst_ptrs;
-    for (auto &dst_buffer_pair : dst_buffers) {
-        dst_ptrs.insert(std::make_pair(dst_buffer_pair.first,
-            MemoryView(reinterpret_cast<uint8_t*>(dst_buffer_pair.second.mutable_data()),
-                dst_buffer_pair.second.nbytes())));
-    }
-
-    const auto src_str = static_cast<std::string>(src);
-    auto status = demuxer.transform_demux(
-        MemoryView(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(src_str.c_str())), src_str.length()), dst_ptrs);
-    VALIDATE_STATUS(status);
-}
-
-void PyhailortInternal::transform_input_buffer(
-    py::array src, const hailo_format_t &src_format, const hailo_3d_image_shape_t &src_shape,
-    uintptr_t dst, size_t dst_size, const hailo_format_t &dst_format, const hailo_3d_image_shape_t &dst_shape,
-    const std::vector<hailo_quant_info_t> &dst_quant_infos)
-{
-    auto transform_context = InputTransformContext::create(src_shape, src_format, dst_shape, dst_format,
-        dst_quant_infos);
-    VALIDATE_EXPECTED(transform_context);
-
-    MemoryView dst_buffer(reinterpret_cast<uint8_t*>(dst), dst_size);
-    auto status = transform_context.value()->transform(
-        MemoryView::create_const(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(src.data())), src.nbytes()),
-        dst_buffer);
-    VALIDATE_STATUS(status);
-}
-
-void PyhailortInternal::transform_output_buffer(
-    py::bytes src, const hailo_format_t &src_format, const hailo_3d_image_shape_t &src_shape,
-    py::array dst, const hailo_format_t &dst_format, const hailo_3d_image_shape_t &dst_shape,
-    const std::vector<hailo_quant_info_t> &dst_quant_infos)
-{
-    auto transform_context = OutputTransformContext::create(src_shape, src_format, dst_shape, dst_format,
-        dst_quant_infos, {});
-    VALIDATE_EXPECTED(transform_context);
-
-    const auto src_str = static_cast<std::string>(src);
-    MemoryView dst_buffer(reinterpret_cast<uint8_t*>(dst.mutable_data()), dst.nbytes());
-    auto status = transform_context.value()->transform(MemoryView::create_const(src_str.c_str(),
-        src_str.length()), dst_buffer);
-    VALIDATE_STATUS(status);
-}
-
-void PyhailortInternal::transform_output_buffer_nms(
-    py::bytes src, const hailo_format_t &src_format, const hailo_3d_image_shape_t &src_shape,
-    py::array dst, const hailo_format_t &dst_format, const hailo_3d_image_shape_t &dst_shape,
-    const std::vector<hailo_quant_info_t> &dst_quant_infos, const hailo_nms_info_t &nms_info)
-{
-    auto transform_context = OutputTransformContext::create(src_shape, src_format, dst_shape, dst_format,
-        dst_quant_infos, nms_info);
-    VALIDATE_EXPECTED(transform_context);
-
-    const auto src_str = static_cast<std::string>(src);
-    MemoryView dst_buffer(reinterpret_cast<uint8_t*>(dst.mutable_data()), dst.nbytes());
-    auto status = transform_context.value()->transform(MemoryView::create_const(src_str.c_str(),
-        src_str.size()), dst_buffer);
-    VALIDATE_STATUS(status);
-}
-
-bool PyhailortInternal::is_input_transformation_required(
-    const hailo_3d_image_shape_t &src_shape, const hailo_format_t &src_format,
-    const hailo_3d_image_shape_t &dst_shape, const hailo_format_t &dst_format,
-    const std::vector<hailo_quant_info_t> &quant_infos)
-{
-    auto expected_is_transforamtion_required = InputTransformContext::is_transformation_required(src_shape, src_format, dst_shape, dst_format,
-        quant_infos);
-    VALIDATE_EXPECTED(expected_is_transforamtion_required);
-
-    return expected_is_transforamtion_required.release();
-}
-
-bool PyhailortInternal::is_output_transformation_required(
-    const hailo_3d_image_shape_t &src_shape, const hailo_format_t &src_format,
-    const hailo_3d_image_shape_t &dst_shape, const hailo_format_t &dst_format,
-    const std::vector<hailo_quant_info_t> &quant_infos)
-{
-    auto expected_is_transforamtion_required = OutputTransformContext::is_transformation_required(src_shape, src_format, dst_shape, dst_format,
-        quant_infos);
-    VALIDATE_EXPECTED(expected_is_transforamtion_required);
-
-    return expected_is_transforamtion_required.release();
-}
-
-py::list PyhailortInternal::get_all_layers_info(const HefWrapper &hef, const std::string &net_group_name)
-{
-    auto core_op_metadata = hef.hef_ptr()->pimpl->get_core_op_metadata(net_group_name);
-    VALIDATE_EXPECTED(core_op_metadata);
-
-    return py::cast(core_op_metadata.value()->get_all_layer_infos());
-}
-
-PYBIND11_MODULE(_pyhailort_internal, m) {
-    ControlWrapper::add_to_python_module(m);
-    m.def("get_yolov5_post_process_expected_buffer", &PyhailortInternal::get_yolov5_post_process_expected_buffer);
-    m.def("demux_output_buffer", &PyhailortInternal::demux_output_buffer);
-    m.def("transform_input_buffer", &PyhailortInternal::transform_input_buffer);
-    m.def("transform_output_buffer", &PyhailortInternal::transform_output_buffer);
-    m.def("transform_output_buffer_nms", &PyhailortInternal::transform_output_buffer_nms);
-    m.def("is_input_transformation_required", &PyhailortInternal::is_input_transformation_required);
-    m.def("is_output_transformation_required", &PyhailortInternal::is_output_transformation_required);
-    m.def("get_all_layers_info", &PyhailortInternal::get_all_layers_info);
-
-    py::class_<BufferIndices>(m, "BufferIndices", py::module_local())
-        .def_readonly("index", &BufferIndices::index)
-        .def_readonly("cluster_index", &BufferIndices::cluster_index)
-        ;
-
-    py::enum_<SENSOR_CONFIG_OPCODES_t>(m, "SensorConfigOpCode")
-        .value("SENSOR_CONFIG_OPCODES_WR", SENSOR_CONFIG_OPCODES_WR)
-        .value("SENSOR_CONFIG_OPCODES_RD", SENSOR_CONFIG_OPCODES_RD)
-        .value("SENSOR_CONFIG_OPCODES_RMW", SENSOR_CONFIG_OPCODES_RMW)
-        .value("SENSOR_CONFIG_OPCODES_DELAY", SENSOR_CONFIG_OPCODES_DELAY)
-        ;
-
-    py::class_<LayerInfo>(m, "HailoLayerInfo", py::module_local())
-        .def_readonly("is_mux", &LayerInfo::is_mux)
-        .def_readonly("mux_predecessors", &LayerInfo::predecessor)
-        .def_readonly("is_multi_planar", &LayerInfo::is_multi_planar)
-        .def_readonly("planes", &LayerInfo::planes)
-        .def_readonly("plane_index", &LayerInfo::plane_index)
-        .def_readonly("is_defused_nms", &LayerInfo::is_defused_nms)
-        .def_readonly("fused_nms_layer", &LayerInfo::fused_nms_layer)
-        .def_property_readonly("shape", [](LayerInfo& self)
-        {
-            switch (self.format.order) {
-                case HAILO_FORMAT_ORDER_NC:
-                    return py::make_tuple(self.shape.features);
-                case HAILO_FORMAT_ORDER_NHW:
-                    return py::make_tuple(self.shape.height, self.shape.width);
-                default:
-                    return py::make_tuple(self.shape.height, self.shape.width, self.shape.features);
-            }
-        })
-        .def_property_readonly("height", [](LayerInfo& self)
-        {
-            return self.shape.height;
-        })
-        .def_property_readonly("width", [](LayerInfo& self)
-        {
-            return self.shape.width;
-        })
-        .def_property_readonly("features", [](LayerInfo& self)
-        {
-            return self.shape.features;
-        })
-        .def("hw_shape", [](LayerInfo& self)
-        {
-            return py::make_tuple(self.hw_shape.height, self.hw_shape.width, self.hw_shape.features);
-        })
-        .def_property_readonly("padded_height", [](LayerInfo& self)
-        {
-            return self.hw_shape.height;
-        })
-        .def_property_readonly("padded_width", [](LayerInfo& self)
-        {
-            return self.hw_shape.width;
-        })
-        .def_property_readonly("padded_features", [](LayerInfo& self)
-        {
-            return self.hw_shape.features;
-        })
-        .def_readonly("data_bytes", &LayerInfo::hw_data_bytes)
-        .def_readonly("format", &LayerInfo::format)
-        .def_property_readonly("format_order", [](LayerInfo& self)
-        {
-            return self.format.order;
-        })
-        .def_readonly("direction", &LayerInfo::direction)
-        .def_readonly("sys_index", &LayerInfo::stream_index)
-        .def_readonly("name", &LayerInfo::name)
-        .def_readonly("quant_infos", &LayerInfo::quant_infos)
-        // For backwards compatibility (accessing qp through layer_info directly)
-        .def_property_readonly("qp_zp", [](LayerInfo& self)
-        {
-            return self.quant_info.qp_zp;
-        })
-        .def_property_readonly("qp_scale", [](LayerInfo& self)
-        {
-            return self.quant_info.qp_scale;
-        })
-        .def_property_readonly("limvals_min", [](LayerInfo& self)
-        {
-            return self.quant_info.limvals_min;
-        })
-        .def_property_readonly("limvals_max", [](LayerInfo& self)
-        {
-            return self.quant_info.limvals_max;
-        })
-        .def_readonly("nms_info", &LayerInfo::nms_info)
-        .def_readonly("height_gcd", &LayerInfo::height_gcd)
-        .def_readonly("height_ratios", &LayerInfo::height_ratios)
-        .def_readonly("buffer_indices", &LayerInfo::buffer_indices)
-        .def_property_readonly("core_bytes_per_buffer", [](LayerInfo& self)
-        {
-            return self.nn_stream_config.core_bytes_per_buffer;
-        })
-        .def_property_readonly("core_buffers_per_frame", [](LayerInfo& self)
-        {
-            return self.nn_stream_config.core_buffers_per_frame;
-        })
-        .def_readonly("network_name", &LayerInfo::network_name)
-        ;
-}
-
-} /* namespace hailort */

hailort/libhailort/bindings/python/src/internal/pyhailort_internal.hpp

@@ -1,53 +0,0 @@
-/**
- * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
- * Distributed under the MIT license (https://opensource.org/licenses/MIT)
- **/
-/**
- * @file pyhailort_internal.hpp
- * @brief Defines binding of internal functions over Python.
- **/
-
-#ifndef _PYHAILORT_INTERNAL_
-#define _PYHAILORT_INTERNAL_
-
-#include "hef/hef_internal.hpp"
-
-#include "hef_api.hpp"
-#include "utils.hpp"
-#include "utils.h"
-#include <pybind11/pybind11.h>
-#include <pybind11/numpy.h>
-#include <pybind11/detail/common.h>
-#include <pybind11/stl.h>
-#include <pybind11/complex.h>
-#include <pybind11/functional.h>
-#include <vector>
-
-
-namespace hailort
-{
-
-class PyhailortInternal {
-public:
-    static py::array get_yolov5_post_process_expected_buffer();
-    static void demux_output_buffer(py::bytes src, const hailo_format_t &src_format, const hailo_3d_image_shape_t &src_shape,
-        std::map<std::string, py::array> dst_buffers, const LayerInfo &mux_layer_info);
-    static void transform_input_buffer(py::array src, const hailo_format_t &src_format, const hailo_3d_image_shape_t &src_shape,
-        uintptr_t dst, size_t dst_size, const hailo_format_t &dst_format, const hailo_3d_image_shape_t &dst_shape,
-        const std::vector<hailo_quant_info_t> &dst_quant_infos);
-    static void transform_output_buffer(py::bytes src, const hailo_format_t &src_format,
-        const hailo_3d_image_shape_t &src_shape, py::array dst, const hailo_format_t &dst_format,
-        const hailo_3d_image_shape_t &dst_shape, const std::vector<hailo_quant_info_t> &dst_quant_infos);
-    static void transform_output_buffer_nms(py::bytes src, const hailo_format_t &src_format,
-        const hailo_3d_image_shape_t &src_shape, py::array dst, const hailo_format_t &dst_format,
-        const hailo_3d_image_shape_t &dst_shape, const std::vector<hailo_quant_info_t> &dst_quant_infos, const hailo_nms_info_t &nms_info);
-    static bool is_input_transformation_required(const hailo_3d_image_shape_t &src_shape, const hailo_format_t &src_format,
-        const hailo_3d_image_shape_t &dst_shape, const hailo_format_t &dst_format, const std::vector<hailo_quant_info_t> &quant_infos);
-    static bool is_output_transformation_required(const hailo_3d_image_shape_t &src_shape, const hailo_format_t &src_format,
-        const hailo_3d_image_shape_t &dst_shape, const hailo_format_t &dst_format, const std::vector<hailo_quant_info_t> &quant_infos);
-    static py::list get_all_layers_info(const HefWrapper &hef, const std::string &net_group_name);
-};
-
-} /* namespace hailort */
-
-#endif /* _PYHAILORT_INTERNAL_ */

hailort/libhailort/bindings/python/src/network_group_api.cpp

@@ -71,7 +71,7 @@ void ActivatedAppContextManagerWrapper::add_to_python_module(py::module &m)
     ;
 
     py::class_<ActivatedNetworkGroup>(m, "ActivatedNetworkGroup")
-        .def("get_intermediate_buffer", [](ActivatedNetworkGroup& self, uint8_t src_context_index,
+        .def("get_intermediate_buffer", [](ActivatedNetworkGroup& self, uint16_t src_context_index,
             uint8_t src_stream_index)
         {
             auto buff = self.get_intermediate_buffer(std::make_pair(src_context_index, src_stream_index));

hailort/libhailort/bindings/python/src/pyhailort.cpp

@@ -501,19 +501,19 @@ PYBIND11_MODULE(_pyhailort, m) {
         .def(py::init<>())
         .def_readonly("number_of_classes", &hailo_nms_shape_t::number_of_classes)
         .def_readonly("max_bboxes_per_class", &hailo_nms_shape_t::max_bboxes_per_class)
-        .def_readonly("max_mask_size", &hailo_nms_shape_t::max_mask_size)
+        .def_readonly("max_accumulated_mask_size", &hailo_nms_shape_t::max_accumulated_mask_size)
         .def(py::pickle(
             [](const hailo_nms_shape_t &nms_shape) { // __getstate__
                 return py::make_tuple(
                     nms_shape.number_of_classes,
                     nms_shape.max_bboxes_per_class,
-                    nms_shape.max_mask_size);
+                    nms_shape.max_accumulated_mask_size);
             },
             [](py::tuple t) { // __setstate__
                 hailo_nms_shape_t nms_shape;
                 nms_shape.number_of_classes = t[0].cast<uint32_t>();
                 nms_shape.max_bboxes_per_class = t[1].cast<uint32_t>();
-                nms_shape.max_mask_size = t[2].cast<uint32_t>();
+                nms_shape.max_accumulated_mask_size = t[2].cast<uint32_t>();
                 return nms_shape;
             }
         ))

hailort/libhailort/bindings/python/src/vstream_api.cpp

@@ -155,7 +155,7 @@ void OutputVStreamWrapper::add_to_python_module(py::module &m)
         // Note: The ownership of the buffer is transferred to Python wrapped as a py::array.
         //       When the py::array isn't referenced anymore in Python and is destructed, the py::capsule's dtor
         //       is called too (and it deletes the raw buffer)
-        auto unmanaged_addr_exp = buffer->storage().release();
+        auto unmanaged_addr_exp = buffer->release();
         VALIDATE_EXPECTED(unmanaged_addr_exp);
         const auto unmanaged_addr = unmanaged_addr_exp.release();
         return py::array(get_dtype(self), get_shape(self), unmanaged_addr,
@@ -176,6 +176,11 @@ void OutputVStreamWrapper::add_to_python_module(py::module &m)
         hailo_status status = self.set_nms_max_proposals_per_class(max_proposals_per_class);
         VALIDATE_STATUS(status);
     })
+    .def("set_nms_max_accumulated_mask_size", [](OutputVStream &self, uint32_t max_accumulated_mask_size)
+    {
+        hailo_status status = self.set_nms_max_accumulated_mask_size(max_accumulated_mask_size);
+        VALIDATE_STATUS(status);
+    })
     .def_property_readonly("info", [](OutputVStream &self)
     {
         return self.get_info();
@@ -403,6 +408,10 @@ void InferVStreamsWrapper::add_to_python_module(py::module &m)
     {
         VALIDATE_STATUS(self.m_infer_pipeline->set_nms_max_proposals_per_class(max_proposals_per_class));
     })
+    .def("set_nms_max_accumulated_mask_size", [](InferVStreamsWrapper &self, uint32_t max_accumulated_mask_size)
+    {
+        VALIDATE_STATUS(self.m_infer_pipeline->set_nms_max_accumulated_mask_size(max_accumulated_mask_size));
+    })
     ;
 }
 

hailort/libhailort/examples/README.md

@@ -57,8 +57,8 @@ The following examples are provided, demonstrating the HailoRT API:
     - For Windows, in case of restricted execution policy, either change the policy, or run the script with "PowerShell -NoProfile -ExecutionPolicy Bypass -File <FilePath>"
   - `notification_callback_example` - Demonstrates how to work with notification callbacks, same as `notification_callback_example` C example.
 You can find more details about each example in the HailoRT user guide.
-  - `async_infer_example` - Basic asynchronous inference of a shortcut network, uses HailoRT C++ api.
+  - `async_infer_basic_example` - Basic asynchronous inference of a multiple input and output model, uses HailoRT C++ api.
-  - `async_infer_functionality_example` - More advanced asynchronous inference of a multiple input and output model, uses HailoRT C++ api.
+  - `async_infer_advanced_example` - More advanced asynchronous inference of a multi planar model, uses HailoRT C++ api.
 ## Compiling with CMake
 Examples are configured and compiled using the following commands:
 ```sh
@@ -87,3 +87,11 @@ To run an example, use (from this examples directory):
   ```sh
   build/<c/cpp>/<example_name>/<example_name> [params..]
   ```
+
+## Hailo Application Code Examples
+
+The examples in this page are for demonstrating HailoRT API usage.
+
+Hailo also offers an additional set of
+[Application Code Examples](https://github.com/hailo-ai/Hailo-Application-Code-Examples),
+which are more application-oriented.

hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(data_quantization_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/infer_pipeline_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(infer_pipeline_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/multi_device_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(multi_device_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/multi_network_vstream_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 find_package(Threads REQUIRED)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(multi_network_vstream_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/notification_callback_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(notification_callback_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/power_measurement_example/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0.0)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(power_measurement_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/raw_async_streams_single_thread_example/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0.0)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(raw_async_streams_single_thread_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/raw_async_streams_single_thread_example/raw_async_streams_single_thread_example.c

@@ -50,11 +50,11 @@ static void output_done_callback(const hailo_stream_read_async_completion_info_t
         // Real applications can forward the buffer to post-process/display. Here we just re-launch new async reads.
         status = hailo_stream_read_raw_buffer_async(stream, completion_info->buffer_addr, completion_info->buffer_size,
             output_done_callback, stream);
-        if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORTED_BY_USER != status)) {
+        if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORT != status)) {
             fprintf(stderr, "Failed read async with status=%d\n", status);
         }
         break;
-    case HAILO_STREAM_ABORTED_BY_USER:
+    case HAILO_STREAM_ABORT:
         // Transfer was canceled, finish gracefully.
         break;
     default:
@@ -73,11 +73,11 @@ static void input_done_callback(const hailo_stream_write_async_completion_info_t
         // new async writes.
         status = hailo_stream_write_raw_buffer_async(stream, completion_info->buffer_addr, completion_info->buffer_size,
             input_done_callback, stream);
-        if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORTED_BY_USER != status)) {
+        if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORT != status)) {
             fprintf(stderr, "Failed write async with status=%d\n", status);
         }
         break;
-    case HAILO_STREAM_ABORTED_BY_USER:
+    case HAILO_STREAM_ABORT:
         // Transfer was canceled, finish gracefully.
         break;
     default:
@@ -85,7 +85,13 @@ static void input_done_callback(const hailo_stream_write_async_completion_info_t
     }
 }
 
-static hailo_status infer(hailo_configured_network_group network_group, size_t number_input_streams,
+typedef struct {
+    void *addr;
+    size_t size;
+    hailo_dma_buffer_direction_t direction;
+} allocated_buffer_t;
+
+static hailo_status infer(hailo_device device, hailo_configured_network_group network_group, size_t number_input_streams,
     hailo_input_stream *input_streams, size_t number_output_streams, hailo_output_stream *output_streams,
     size_t ongoing_transfers)
 {
@@ -95,7 +101,8 @@ static hailo_status infer(hailo_configured_network_group network_group, size_t n
     size_t frame_size = 0;
     size_t stream_index = 0;
     void *current_buffer = NULL;
-    void *buffers[MAX_EDGE_LAYERS * MAX_ONGOING_TRANSFERS] = {0};
+
+    allocated_buffer_t buffers[MAX_EDGE_LAYERS * MAX_ONGOING_TRANSFERS] = {0};
     size_t allocated_buffers = 0;
 
     // We launch "ongoing_transfers" async operations for both input and output streams. On each async callback, we launch
@@ -108,7 +115,12 @@ static hailo_status infer(hailo_configured_network_group network_group, size_t n
             // Buffers read from async operation must be page aligned.
             current_buffer = page_aligned_alloc(frame_size);
             REQUIRE_ACTION(INVALID_ADDR != current_buffer, status=HAILO_OUT_OF_HOST_MEMORY, l_shutdown, "allocation failed");
-            buffers[allocated_buffers++] = current_buffer;
+            buffers[allocated_buffers++] = (allocated_buffer_t){ current_buffer, frame_size, HAILO_DMA_BUFFER_DIRECTION_D2H };
+
+            // If the same buffer is used multiple times on async-io, to improve performance, it is recommended to
+            // pre-map it into the device.
+            status = hailo_device_dma_map_buffer(device, current_buffer, frame_size, HAILO_DMA_BUFFER_DIRECTION_D2H);
+            REQUIRE_SUCCESS(status, l_shutdown, "Failed map buffer with status=%d", status);
 
             status = hailo_stream_read_raw_buffer_async(output_streams[stream_index], current_buffer, frame_size,
                 output_done_callback, output_streams[stream_index]);
@@ -124,7 +136,12 @@ static hailo_status infer(hailo_configured_network_group network_group, size_t n
             // Buffers written to async operation must be page aligned.
             current_buffer = page_aligned_alloc(frame_size);
             REQUIRE_ACTION(INVALID_ADDR != current_buffer, status=HAILO_OUT_OF_HOST_MEMORY, l_shutdown, "allocation failed");
-            buffers[allocated_buffers++] = current_buffer;
+            buffers[allocated_buffers++] = (allocated_buffer_t){ current_buffer, frame_size, HAILO_DMA_BUFFER_DIRECTION_H2D };
+
+            // If the same buffer is used multiple times on async-io, to improve performance, it is recommended to
+            // pre-map it into the device.
+            status = hailo_device_dma_map_buffer(device, current_buffer, frame_size, HAILO_DMA_BUFFER_DIRECTION_H2D);
+            REQUIRE_SUCCESS(status, l_shutdown, "Failed map buffer with status=%d", status);
 
             status = hailo_stream_write_raw_buffer_async(input_streams[stream_index], current_buffer, frame_size,
                 input_done_callback, input_streams[stream_index]);
@@ -138,11 +155,14 @@ static hailo_status infer(hailo_configured_network_group network_group, size_t n
     status = HAILO_SUCCESS;
 l_shutdown:
     // Calling hailo_shutdown_network_group will ensure that all async operations are done. All pending async I/O
-    // operations will be canceled and their callbacks called with status=HAILO_STREAM_ABORTED_BY_USER.
+    // operations will be canceled and their callbacks called with status=HAILO_STREAM_ABORT.
     (void) hailo_shutdown_network_group(network_group);
 
     // There are no async I/O operations ongoing so it is safe to free the buffers now.
-    for (i = 0; i < allocated_buffers; i++) page_aligned_free(buffers[i], frame_size);
+    for (i = 0; i < allocated_buffers; i++) {
+        (void) hailo_device_dma_unmap_buffer(device, buffers[i].addr, buffers[i].size, buffers[i].direction);
+        page_aligned_free(buffers[i].addr, buffers[i].size);
+    }
 
     return status;
 }
@@ -239,7 +259,7 @@ int main()
     REQUIRE_SUCCESS(status, l_release_device, "Failed activate network group");
 
     // Run infer.
-    status = infer(network_group, number_input_streams, input_streams, number_output_streams, output_streams,
+    status = infer(device, network_group, number_input_streams, input_streams, number_output_streams, output_streams,
         ongoing_transfers);
     REQUIRE_SUCCESS(status, l_deactivate, "Failed performing inference");
 

hailort/libhailort/examples/c/raw_streams_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(raw_streams_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/switch_network_groups_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(switch_network_groups_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/switch_network_groups_example/switch_network_groups_example.c

@@ -192,7 +192,7 @@ int main()
     write_thread_args_t write_args[HEF_COUNT][MAX_EDGE_LAYERS];
     read_thread_args_t read_args[HEF_COUNT][MAX_EDGE_LAYERS];
 
-    char HEF_FILES[HEF_COUNT][MAX_HEF_PATH_LEN] = {"hefs/multi_network_shortcut_net.hef", "hefs/shortcut_net.hef"};
+    char HEF_FILES[HEF_COUNT][MAX_HEF_PATH_LEN] = {"hefs/shortcut_net_nv12.hef", "hefs/shortcut_net.hef"};
     // Note: default batch_size is 0, which is not used in this example
     uint16_t batch_sizes[HEF_COUNT] = {BATCH_SIZE_1, BATCH_SIZE_2};
 

hailort/libhailort/examples/c/switch_network_groups_manually_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(switch_network_groups_manually_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/c/vstreams_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(vstreams_example.c PROPERTIES LANGUAGE C)
 

hailort/libhailort/examples/cpp/CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.0.0)
 
 add_subdirectory(vstreams_example)
 add_subdirectory(infer_pipeline_example)
-add_subdirectory(async_infer_example)
+add_subdirectory(async_infer_basic_example)
-add_subdirectory(async_infer_functionality_example)
+add_subdirectory(async_infer_advanced_example)
 add_subdirectory(raw_streams_example)
 add_subdirectory(multi_network_vstream_example)
 add_subdirectory(switch_network_groups_example)
@@ -17,8 +17,8 @@ add_subdirectory(notification_callback_example)
 set(CPP_EXAMPLE_TARGETS
     cpp_vstreams_example
     cpp_infer_pipeline_example
-    cpp_async_infer_example
+    cpp_async_infer_basic_example
-    cpp_async_infer_functionality_example
+    cpp_async_infer_advanced_example
     cpp_raw_streams_example
     cpp_multi_network_vstream_example
     cpp_switch_network_groups_example

hailort/libhailort/examples/cpp/async_infer_advanced_example/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.0.0)
+
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
+
+add_executable(cpp_async_infer_advanced_example async_infer_advanced_example.cpp)
+target_link_libraries(cpp_async_infer_advanced_example PRIVATE HailoRT::libhailort)
+
+if(WIN32)
+    target_compile_options(cpp_async_infer_advanced_example PRIVATE
+        /DWIN32_LEAN_AND_MEAN
+        /DNOMINMAX                  # NOMINMAX is required in order to play nice with std::min/std::max (otherwise Windows.h defines it's own)
+        /wd4201 /wd4251
+    )
+endif()
+
+set_target_properties(cpp_async_infer_advanced_example PROPERTIES CXX_STANDARD 14)

hailort/libhailort/examples/cpp/async_infer_advanced_example/async_infer_advanced_example.cpp

@@ -3,9 +3,10 @@
  * Distributed under the MIT license (https://opensource.org/licenses/MIT)
  **/
 /**
- * @file async_infer_functionality_example.cpp
+ * @file async_infer_advanced_example.cpp
- * This example demonstrates the Async Infer API usage with a specific model with multiple inputs and outputs
+ * This example demonstrates the Async Infer API usage with a specific model that has multi-planar input
  * and changes configutrations of the streams.
+ * Multiple infer jobs are triggered, and waiting for the last one ensures that all the rest will arrive as well.
  **/
 
 #include "hailo/hailort.hpp"
@@ -43,46 +44,67 @@ int main()
         return vdevice.status();
     }
 
-    auto infer_model_exp = vdevice.value()->create_infer_model("hefs/multi_network_shortcut_net.hef");
+    // Create infer model from HEF file.
+    auto infer_model_exp = vdevice.value()->create_infer_model("hefs/shortcut_net_nv12.hef");
     if (!infer_model_exp) {
         std::cerr << "Failed to create infer model, status = " << infer_model_exp.status() << std::endl;
         return infer_model_exp.status();
     }
     auto infer_model = infer_model_exp.release();
 
-    infer_model->input("multi_network_shortcut_net_scope1/input_layer_0")->set_format_type(HAILO_FORMAT_TYPE_FLOAT32);
+    infer_model->output()->set_format_type(HAILO_FORMAT_TYPE_FLOAT32);
-    infer_model->output("multi_network_shortcut_net_scope1/shortcut0")->set_format_type(HAILO_FORMAT_TYPE_FLOAT32);
-    infer_model->input("multi_network_shortcut_net_scope2/input_layer_1")->set_format_type(HAILO_FORMAT_TYPE_FLOAT32);
-    infer_model->output("multi_network_shortcut_net_scope2/shortcut1")->set_format_type(HAILO_FORMAT_TYPE_FLOAT32);
 
+    // Configure the infer model
     auto configured_infer_model = infer_model->configure();
     if (!configured_infer_model) {
         std::cerr << "Failed to create configured infer model, status = " << configured_infer_model.status() << std::endl;
         return configured_infer_model.status();
     }
 
-    // We store buffers vector here as a guard for the memory. The buffer will be freed only after
+    // The buffers are stored here as a guard for the memory. The buffer will be freed only after
     // configured_infer_model will be released.
     std::vector<std::shared_ptr<uint8_t>> buffer_guards;
 
+    // Create infer bindings
     auto bindings = configured_infer_model->create_bindings();
     if (!bindings) {
         std::cerr << "Failed to create infer bindings, status = " << bindings.status() << std::endl;
         return bindings.status();
     }
 
+    // Set the input buffers of the bindings.
     for (const auto &input_name : infer_model->get_input_names()) {
         size_t input_frame_size = infer_model->input(input_name)->get_frame_size();
-        auto input_buffer = page_aligned_alloc(input_frame_size);
+
-        auto status = bindings->input(input_name)->set_buffer(MemoryView(input_buffer.get(), input_frame_size));
+        // create pix_buffer
+        const auto Y_PLANE_SIZE = static_cast<uint32_t>(input_frame_size * 2 / 3);
+        const auto UV_PLANE_SIZE = static_cast<uint32_t>(input_frame_size * 1 / 3);
+        assert (Y_PLANE_SIZE + UV_PLANE_SIZE == input_frame_size);
+        auto y_plane_buffer = page_aligned_alloc(Y_PLANE_SIZE);
+        auto uv_plane_buffer = page_aligned_alloc(UV_PLANE_SIZE);
+        hailo_pix_buffer_t pix_buffer{};
+        pix_buffer.memory_type = HAILO_PIX_BUFFER_MEMORY_TYPE_USERPTR;
+        pix_buffer.number_of_planes = 2;
+        // Y Plane
+        pix_buffer.planes[0].bytes_used = Y_PLANE_SIZE;
+        pix_buffer.planes[0].plane_size = Y_PLANE_SIZE;
+        pix_buffer.planes[0].user_ptr = reinterpret_cast<void*>(y_plane_buffer.get());
+        // UV Plane
+        pix_buffer.planes[1].bytes_used = UV_PLANE_SIZE;
+        pix_buffer.planes[1].plane_size = UV_PLANE_SIZE;
+        pix_buffer.planes[1].user_ptr = reinterpret_cast<void*>(uv_plane_buffer.get());
+
+        auto status = bindings->input(input_name)->set_pix_buffer(pix_buffer);
         if (HAILO_SUCCESS != status) {
             std::cerr << "Failed to set infer input buffer, status = " << status << std::endl;
             return status;
         }
 
-        buffer_guards.push_back(input_buffer);
+        buffer_guards.push_back(y_plane_buffer);
+        buffer_guards.push_back(uv_plane_buffer);
     }
 
+    // Set the output buffers of the bindings.
     for (const auto &output_name : infer_model->get_output_names()) {
         size_t output_frame_size = infer_model->output(output_name)->get_frame_size();
         auto output_buffer = page_aligned_alloc(output_frame_size);
@@ -111,6 +133,7 @@ int main()
             std::cerr << "Failed to start async infer job, status = " << job.status() << std::endl;
             return job.status();
         }
+        // detach() is called in order for jobs to run in parallel (and not one after the other)
         job->detach();
 
         if (i == FRAMES_COUNT - 1) {
@@ -125,5 +148,6 @@ int main()
         return status;
     }
 
+    std::cout << "Inference finished successfully" << std::endl;
     return HAILO_SUCCESS;
 }

hailort/libhailort/examples/cpp/async_infer_basic_example/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.0.0)
+
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
+
+add_executable(cpp_async_infer_basic_example async_infer_basic_example.cpp)
+target_link_libraries(cpp_async_infer_basic_example PRIVATE HailoRT::libhailort)
+
+if(WIN32)
+    target_compile_options(cpp_async_infer_basic_example PRIVATE
+        /DWIN32_LEAN_AND_MEAN
+        /DNOMINMAX                  # NOMINMAX is required in order to play nice with std::min/std::max (otherwise Windows.h defines it's own)
+        /wd4201 /wd4251
+    )
+endif()
+
+set_target_properties(cpp_async_infer_basic_example PROPERTIES CXX_STANDARD 14)

hailort/libhailort/examples/cpp/async_infer_basic_example/async_infer_basic_example.cpp

@@ -3,8 +3,8 @@
  * Distributed under the MIT license (https://opensource.org/licenses/MIT)
  **/
 /**
- * @file async_infer_example.cpp
+ * @file async_infer_basic_example.cpp
- * This example demonstrates the Async Infer API usage and assumes the model has only one input and output.
+ * This example demonstrates the Async Infer API usage with a specific model.
  **/
 
 #include "hailo/hailort.hpp"
@@ -42,6 +42,7 @@ int main()
         return vdevice.status();
     }
 
+    // Create infer model from HEF file.
     auto infer_model_exp = vdevice.value()->create_infer_model(HEF_FILE);
     if (!infer_model_exp) {
         std::cerr << "Failed to create infer model, status = " << infer_model_exp.status() << std::endl;
@@ -49,45 +50,60 @@ int main()
     }
     auto infer_model = infer_model_exp.release();
 
+    // Configure the infer model
     auto configured_infer_model = infer_model->configure();
     if (!configured_infer_model) {
         std::cerr << "Failed to create configured infer model, status = " << configured_infer_model.status() << std::endl;
         return configured_infer_model.status();
     }
 
+    // The buffers are stored here as a guard for the memory. The buffer will be freed only after
+    // configured_infer_model will be released.
+    std::vector<std::shared_ptr<uint8_t>> buffer_guards;
+
     auto bindings = configured_infer_model->create_bindings();
     if (!bindings) {
         std::cerr << "Failed to create infer bindings, status = " << bindings.status() << std::endl;
         return bindings.status();
     }
 
-    size_t input_frame_size = infer_model->input()->get_frame_size();
+    for (const auto &input_name : infer_model->get_input_names()) {
-    auto input_buffer = page_aligned_alloc(input_frame_size);
+        size_t input_frame_size = infer_model->input(input_name)->get_frame_size();
-    auto status = bindings->input()->set_buffer(MemoryView(input_buffer.get(), input_frame_size));
+        auto input_buffer = page_aligned_alloc(input_frame_size);
-    if (HAILO_SUCCESS != status) {
+        auto status = bindings->input(input_name)->set_buffer(MemoryView(input_buffer.get(), input_frame_size));
-        std::cerr << "Failed to set infer input buffer, status = " << status << std::endl;
+        if (HAILO_SUCCESS != status) {
-        return status;
+            std::cerr << "Failed to set infer input buffer, status = " << status << std::endl;
+            return status;
+        }
+
+        buffer_guards.push_back(input_buffer);
     }
 
-    size_t output_frame_size = infer_model->output()->get_frame_size();
+    for (const auto &output_name : infer_model->get_output_names()) {
-    auto output_buffer = page_aligned_alloc(output_frame_size);
+        size_t output_frame_size = infer_model->output(output_name)->get_frame_size();
-    status = bindings->output()->set_buffer(MemoryView(output_buffer.get(), output_frame_size));
+        auto output_buffer = page_aligned_alloc(output_frame_size);
-    if (HAILO_SUCCESS != status) {
+        auto status = bindings->output(output_name)->set_buffer(MemoryView(output_buffer.get(), output_frame_size));
-        std::cerr << "Failed to set infer input buffer, status = " << status << std::endl;
+        if (HAILO_SUCCESS != status) {
-        return status;
+            std::cerr << "Failed to set infer output buffer, status = " << status << std::endl;
+            return status;
+        }
+
+        buffer_guards.push_back(output_buffer);
     }
 
+    // Run the async infer job.
     auto job = configured_infer_model->run_async(bindings.value());
     if (!job) {
         std::cerr << "Failed to start async infer job, status = " << job.status() << std::endl;
         return job.status();
     }
 
-    status = job->wait(std::chrono::milliseconds(1000));
+    auto status = job->wait(std::chrono::milliseconds(1000));
     if (HAILO_SUCCESS != status) {
         std::cerr << "Failed to wait for infer to finish, status = " << status << std::endl;
         return status;
     }
 
+    std::cout << "Inference finished successfully" << std::endl;
     return HAILO_SUCCESS;
 }

hailort/libhailort/examples/cpp/async_infer_example/CMakeLists.txt

@@ -1,16 +0,0 @@
-cmake_minimum_required(VERSION 3.0.0)
-
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
-
-add_executable(cpp_async_infer_example async_infer_example.cpp)
-target_link_libraries(cpp_async_infer_example PRIVATE HailoRT::libhailort)
-
-if(WIN32)
-    target_compile_options(cpp_async_infer_example PRIVATE
-        /DWIN32_LEAN_AND_MEAN
-        /DNOMINMAX                  # NOMINMAX is required in order to play nice with std::min/std::max (otherwise Windows.h defines it's own)
-        /wd4201 /wd4251
-    )
-endif()
-
-set_target_properties(cpp_async_infer_example PROPERTIES CXX_STANDARD 14)

hailort/libhailort/examples/cpp/async_infer_functionality_example/CMakeLists.txt

@@ -1,16 +0,0 @@
-cmake_minimum_required(VERSION 3.0.0)
-
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
-
-add_executable(cpp_async_infer_functionality_example async_infer_functionality_example.cpp)
-target_link_libraries(cpp_async_infer_functionality_example PRIVATE HailoRT::libhailort)
-
-if(WIN32)
-    target_compile_options(cpp_async_infer_functionality_example PRIVATE
-        /DWIN32_LEAN_AND_MEAN
-        /DNOMINMAX                  # NOMINMAX is required in order to play nice with std::min/std::max (otherwise Windows.h defines it's own)
-        /wd4201 /wd4251
-    )
-endif()
-
-set_target_properties(cpp_async_infer_functionality_example PROPERTIES CXX_STANDARD 14)

hailort/libhailort/examples/cpp/infer_pipeline_example/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0.0)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_infer_pipeline_example infer_pipeline_example.cpp)
 target_link_libraries(cpp_infer_pipeline_example PRIVATE HailoRT::libhailort)

hailort/libhailort/examples/cpp/multi_device_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_multi_device_example multi_device_example.cpp)
 target_link_libraries(cpp_multi_device_example PRIVATE HailoRT::libhailort Threads::Threads)

hailort/libhailort/examples/cpp/multi_network_vstream_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 find_package(Threads REQUIRED)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_multi_network_vstream_example multi_network_vstream_example.cpp)
 target_link_libraries(cpp_multi_network_vstream_example PRIVATE HailoRT::libhailort Threads::Threads)

hailort/libhailort/examples/cpp/multi_process_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_multi_process_example multi_process_example.cpp)
 target_link_libraries(cpp_multi_process_example PRIVATE HailoRT::libhailort Threads::Threads)

hailort/libhailort/examples/cpp/multi_process_example/multi_process_example.ps1

@@ -5,7 +5,7 @@ Param(
 )
 
 $max_processes_count = 8
-$first_hef="hefs\multi_network_shortcut_net.hef"
+$first_hef="hefs\shortcut_net_nv12.hef"
 $second_hef="hefs\shortcut_net.hef"
 $executable_base_name="cpp_multi_process_example"
 $executable_name="$executable_base_name.exe"

hailort/libhailort/examples/cpp/multi_process_example/multi_process_example.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-readonly first_hef="hefs/multi_network_shortcut_net.hef"
+readonly first_hef="hefs/shortcut_net_nv12.hef"
 readonly second_hef="hefs/shortcut_net.hef"
 readonly max_processes_count=8
 readonly default_processes_count=1

hailort/libhailort/examples/cpp/notification_callback_example/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0.0)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_notification_callback_example notification_callback_example.cpp)
 target_link_libraries(cpp_notification_callback_example PRIVATE HailoRT::libhailort)

hailort/libhailort/examples/cpp/power_measurement_example/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0.0)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_power_measurement_example power_measurement_example.cpp)
 target_link_libraries(cpp_power_measurement_example PRIVATE HailoRT::libhailort)

hailort/libhailort/examples/cpp/raw_async_streams_multi_thread_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_raw_async_streams_multi_thread_example raw_async_streams_multi_thread_example.cpp)
 target_link_libraries(cpp_raw_async_streams_multi_thread_example PRIVATE HailoRT::libhailort Threads::Threads)

hailort/libhailort/examples/cpp/raw_async_streams_multi_thread_example/raw_async_streams_multi_thread_example.cpp

@@ -71,8 +71,8 @@ Expected<std::shared_ptr<ConfiguredNetworkGroup>> configure_network_group(Device
 static void output_async_callback(const OutputStream::CompletionInfo &completion_info)
 {
     // Real applications can free the buffer or forward it to post-process/display.
-    if ((HAILO_SUCCESS != completion_info.status) && (HAILO_STREAM_ABORTED_BY_USER != completion_info.status)) {
+    if ((HAILO_SUCCESS != completion_info.status) && (HAILO_STREAM_ABORT != completion_info.status)) {
-        // We will get HAILO_STREAM_ABORTED_BY_USER when activated_network_group is destructed.
+        // We will get HAILO_STREAM_ABORT when activated_network_group is destructed.
         std::cerr << "Got an unexpected status on callback. status=" << completion_info.status << std::endl;
     }
 }
@@ -80,13 +80,13 @@ static void output_async_callback(const OutputStream::CompletionInfo &completion
 static void input_async_callback(const InputStream::CompletionInfo &completion_info)
 {
     // Real applications can free the buffer or reuse it for next transfer.
-    if ((HAILO_SUCCESS != completion_info.status) && (HAILO_STREAM_ABORTED_BY_USER  != completion_info.status)) {
+    if ((HAILO_SUCCESS != completion_info.status) && (HAILO_STREAM_ABORT  != completion_info.status)) {
-        // We will get HAILO_STREAM_ABORTED_BY_USER  when activated_network_group is destructed.
+        // We will get HAILO_STREAM_ABORT  when activated_network_group is destructed.
         std::cerr << "Got an unexpected status on callback. status=" << completion_info.status << std::endl;
     }
 }
 
-static hailo_status infer(ConfiguredNetworkGroup &network_group)
+static hailo_status infer(Device &device, ConfiguredNetworkGroup &network_group)
 {
     // Assume one input and output
     auto &output = network_group.get_output_streams()[0].get();
@@ -101,6 +101,16 @@ static hailo_status infer(ConfiguredNetworkGroup &network_group)
     auto output_buffer = page_aligned_alloc(output.get_frame_size());
     auto input_buffer = page_aligned_alloc(input.get_frame_size());
 
+    // If the same buffer is used multiple times on async-io, to improve performance, it is recommended to pre-map it
+    // into the device. The DmaMappedBuffer object manages the mapping, and it'll be unmapped when it is destroyed.
+    // Notice that the buffer must be alive as long as the mapping is alive, so we define it after the buffers.
+    auto output_mapping = DmaMappedBuffer::create(device, output_buffer.get(), output.get_frame_size(), HAILO_DMA_BUFFER_DIRECTION_D2H);
+    auto input_mapping = DmaMappedBuffer::create(device, input_buffer.get(), input.get_frame_size(), HAILO_DMA_BUFFER_DIRECTION_H2D);
+    if (!output_mapping || !input_mapping) {
+        std::cerr << "Failed to map buffer with status=" << input_mapping.status() << ", " << output_mapping.status() << std::endl;
+        return HAILO_INTERNAL_FAILURE;
+    }
+
     std::atomic<hailo_status> output_status(HAILO_UNINITIALIZED);
     std::thread output_thread([&]() {
         while (true) {
@@ -127,16 +137,16 @@ static hailo_status infer(ConfiguredNetworkGroup &network_group)
     std::this_thread::sleep_for(std::chrono::seconds(5));
 
     // Calling shutdown on a network group will ensure that all async operations are done. All pending
-    // operations will be canceled and their callbacks will be called with status=HAILO_STREAM_ABORTED_BY_USER.
+    // operations will be canceled and their callbacks will be called with status=HAILO_STREAM_ABORT.
     // Only after the shutdown is called, we can safely free the buffers and any variable captured inside the async
     // callback lambda.
     network_group.shutdown();
 
-    // Thread should be stopped with HAILO_STREAM_ABORTED_BY_USER status.
+    // Thread should be stopped with HAILO_STREAM_ABORT status.
     output_thread.join();
     input_thread.join();
 
-    if ((HAILO_STREAM_ABORTED_BY_USER != output_status) || (HAILO_STREAM_ABORTED_BY_USER != input_status)) {
+    if ((HAILO_STREAM_ABORT != output_status) || (HAILO_STREAM_ABORT != input_status)) {
         std::cerr << "Got unexpected statues from thread: " << output_status << ", " << input_status << std::endl;
         return HAILO_INTERNAL_FAILURE;
     }
@@ -165,7 +175,7 @@ int main()
         return EXIT_FAILURE;
     }
 
-    auto status = infer(*network_group.value());
+    auto status = infer(*device.value(), *network_group.value());
     if (HAILO_SUCCESS != status) {
         return EXIT_FAILURE;
     }

hailort/libhailort/examples/cpp/raw_async_streams_single_thread_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-find_package(HailoRT 4.16.2 EXACT REQUIRED)
+find_package(HailoRT 4.17.0 EXACT REQUIRED)
 
 add_executable(cpp_raw_async_streams_single_thread_example raw_async_streams_single_thread_example.cpp)
 target_link_libraries(cpp_raw_async_streams_single_thread_example PRIVATE HailoRT::libhailort Threads::Threads)

hailort/libhailort/examples/cpp/raw_async_streams_single_thread_example/raw_async_streams_single_thread_example.cpp

@@ -61,11 +61,11 @@ static hailo_status infer(ConfiguredNetworkGroup &network_group)
         case HAILO_SUCCESS:
             // Real applications can forward the buffer to post-process/display. Here we just re-launch new async read.
             status = output.read_async(completion_info.buffer_addr, completion_info.buffer_size, read_done);
-            if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORTED_BY_USER != status)) {
+            if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORT != status)) {
                 std::cerr << "Failed read async with status=" << status << std::endl;
             }
             break;
-        case HAILO_STREAM_ABORTED_BY_USER:
+        case HAILO_STREAM_ABORT:
             // Transfer was canceled, finish gracefully.
             break;
         default:
@@ -80,11 +80,11 @@ static hailo_status infer(ConfiguredNetworkGroup &network_group)
             // Real applications may free the buffer and replace it with new buffer ready to be sent. Here we just
             // re-launch new async write.
             status = input.write_async(completion_info.buffer_addr, completion_info.buffer_size, write_done);
-            if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORTED_BY_USER != status)) {
+            if ((HAILO_SUCCESS != status) && (HAILO_STREAM_ABORT != status)) {
                 std::cerr << "Failed read async with status=" << status << std::endl;
             }
             break;
-        case HAILO_STREAM_ABORTED_BY_USER:
+        case HAILO_STREAM_ABORT:
             // Transfer was canceled, finish gracefully.
             break;
         default:
@@ -121,7 +121,7 @@ static hailo_status infer(ConfiguredNetworkGroup &network_group)
     std::this_thread::sleep_for(std::chrono::seconds(5));
 
     // Calling shutdown on a network group will ensure that all async operations are done. All pending
-    // operations will be canceled and their callbacks will be called with status=HAILO_STREAM_ABORTED_BY_USER.
+    // operations will be canceled and their callbacks will be called with status=HAILO_STREAM_ABORT.
     // Only after the shutdown is called, we can safely free the buffers and any variable captured inside the async
     // callback lambda.
     network_group.shutdown();