v4.13.0 (#8)

Co-authored-by: HailoRT-Automation <contact@hailo.ai>

README.md

@@ -5,19 +5,18 @@
 
 # HailoRT #
 
-HailoRT is a light-weight and production-grade run-time library, which runs on the host processor, and
+HailoRT is a lightweight, production-grade runtime library that runs on the host processor and provides a robust
-implements a robust user-space run-time library (HailoRT Library) responsible for operating a Hailo device, with intuitive APIs in C/C++ for optimized performance.
+user-space runtime library (the HailoRT Library) with intuitive APIs in C/C++ for optimized performance
 
-HailoRT is comprised of the following main components:
+HailoRT consists of the following main components:
 - HailoRT Library.
-- HailoRT CLI - command line application used to control the Hailo device, run inference using the device,
+- HailoRT CLI - a command line application used to control the Hailo device, run inferences, collect statistics and device events, etc.
- collect inference statistics and device events, etc.
+- [**HailoRT PCIe Driver**](https://github.com/hailo-ai/hailort-drivers) - the device driver used to manage the Hailo device, communicate with the device,
-- [**HailoRT PCIe Driver**](https://github.com/hailo-ai/hailort-drivers) - the device driver used to manage the Hailo device, communicate with the device and transfer
+and transfer data to/from the device; it includes the Hailo-8 firmware that runs on the Hailo device and manages its boot and control.
-    data to/from the device. The PCIe driver includes the Hailo-8 firmware that runs on the Hailo device, manages the boot and control of the Hailo device.
+- pyHailoRT - HailoRT Python API, which wraps the runtime library.
-- pyHailoRT - HailoRT Python API (wraps the run-time library)
 - HailoRT GStreamer element (HailoNet).
 
-HailoRT supports Linux and Windows, and can be compiled from sources to be integrated with various x86 and ARM processors.
+HailoRT supports Linux and Windows, and it can be compiled from sources to be integrated with various x86 and ARM processors.
 
 ## Usage
 

common/include/control_protocol.h

@@ -41,7 +41,10 @@ extern "C" {
 #define CONTROL_PROTOCOL__SOC_ID_LENGTH (32)
 #define CONTROL_PROTOCOL__MAX_CFG_CHANNELS (4)
 #define CONTROL_PROTOCOL__MAX_NETWORKS_PER_NETWORK_GROUP (8)
+#define CONTROL_PROTOCOL__MAX_VDMA_CHANNELS_PER_ENGINE (32)
 #define CONTROL_PROTOCOL__MAX_VDMA_ENGINES_COUNT (3)
+#define CONTROL_PROTOCOL__MAX_TOTAL_CHANNEL_COUNT \
+    (CONTROL_PROTOCOL__MAX_VDMA_CHANNELS_PER_ENGINE * CONTROL_PROTOCOL__MAX_VDMA_ENGINES_COUNT)
 /* Tightly coupled with the sizeof PROCESS_MONITOR__detection_results_t 
     and HAILO_SOC_PM_VALUES_BYTES_LENGTH */
 #define PM_RESULTS_LENGTH (24)
@@ -154,6 +157,7 @@ extern "C" {
     CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CONTEXT_SWITCH_CLEAR_CONFIGURED_APPS,      false, CPU_ID_CORE_CPU)\
     CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_GET_HW_CONSTS,                             false, CPU_ID_CORE_CPU)\
     CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_SET_SLEEP_STATE,                           false, CPU_ID_APP_CPU)\
+    CONTROL_PROTOCOL__OPCODE_X(HAILO_CONTROL_OPCODE_CHANGE_HW_INFER_STATUS,                    false, CPU_ID_CORE_CPU)\
 
 typedef enum {
 #define CONTROL_PROTOCOL__OPCODE_X(name, is_critical, cpu_id) name,
@@ -340,8 +344,7 @@ typedef enum {
     CONTROL_PROTOCOL__HAILO8_A0 = 0,
     CONTROL_PROTOCOL__HAILO8,
     CONTROL_PROTOCOL__HAILO8L,
-    CONTROL_PROTOCOL__MERCURY_CA,
+    CONTROL_PROTOCOL__HAILO15,
-    CONTROL_PROTOCOL__MERCURY_VPU,
     /* Must be last!! */
     CONTROL_PROTOCOL__DEVICE_ARCHITECTURE_COUNT
 } CONTROL_PROTOCOL__device_architecture_t;
@@ -868,7 +871,9 @@ typedef struct {
     CONTROL_PROTOCOL__INFER_FEATURE_LIST_t infer_features;
     CONTROL_PROTOCOL__VALIDATION_FEATURE_LIST_t validation_features;
     uint8_t networks_count;
+    uint16_t csm_buffer_size;
     uint16_t batch_size[CONTROL_PROTOCOL__MAX_NETWORKS_PER_NETWORK_GROUP];
+    uint32_t boundary_channels_bitmap[CONTROL_PROTOCOL__MAX_VDMA_ENGINES_COUNT];
 } CONTROL_PROTOCOL__application_header_t;
 
 typedef struct {
@@ -933,7 +938,8 @@ typedef struct {
     uint8_t buffer_type;   // CONTROL_PROTOCOL__HOST_BUFFER_TYPE_t
     uint64_t dma_address;
     uint16_t desc_page_size;
-    uint32_t total_desc_count;
+    uint32_t total_desc_count; //HRT-9913 - Some descriptors may not be initialized (to save space), needs to
+                               // change this param or add another one for validation.
     uint32_t bytes_in_pattern;
 } CONTROL_PROTOCOL__host_buffer_info_t;
 
@@ -950,8 +956,6 @@ typedef struct {
     uint8_t is_last_control_per_context;
     uint32_t context_type_length;
     uint8_t context_type; // CONTROL_PROTOCOL__context_switch_context_type_t
-    uint32_t actions_count_length;
-    uint32_t actions_count;
     uint32_t context_network_data_length;
     uint8_t context_network_data[0];
 } CONTROL_PROTOCOL__context_switch_set_context_info_request_t;
@@ -959,11 +963,6 @@ typedef struct {
 #pragma warning(pop)
 #endif
 
-typedef struct {
-    /* Must be first */
-    uint8_t action_type; // CONTEXT_SWITCH_DEFS__ACTION_TYPE_t
-    bool is_repeated;
-} CONTROL_PROTOCOL__ACTION_HEADER_t;
 
 typedef CONTROL_PROTOCOL__read_memory_request_t CONTROL_PROTOCOL__read_user_config_request_t;
 typedef CONTROL_PROTOCOL__read_memory_response_t CONTROL_PROTOCOL__read_user_config_response_t;
@@ -1279,6 +1278,47 @@ typedef struct {
     CONTROL_PROTOCOL__hw_consts_t hw_consts;
 } CONTROL_PROTOCOL__get_hw_consts_response_t;
 
+/* TODO HRT-9545 - Return and hw only parse results */
+typedef struct {
+    bool infer_done;
+    uint32_t infer_cycles;
+} CONTROL_PROTOCOL__hw_only_infer_results_t;
+
+typedef struct {
+    uint32_t results_length;
+    CONTROL_PROTOCOL__hw_only_infer_results_t results;
+} CONTROL_PROTOCOL__change_hw_infer_status_response_t;
+
+typedef struct {
+    uint8_t channel_index;
+    uint8_t engine_index;
+    uint16_t desc_programed;
+} CONTROL_PROTOCOL__hw_infer_channel_info_t;
+
+typedef struct {
+    CONTROL_PROTOCOL__hw_infer_channel_info_t channel_info[CONTROL_PROTOCOL__MAX_TOTAL_CHANNEL_COUNT];
+    uint8_t channel_count;
+} CONTROL_PROTOCOL__hw_infer_channels_info_t;
+
+typedef enum {
+    CONTROL_PROTOCOL__HW_INFER_STATE_START, 
+    CONTROL_PROTOCOL__HW_INFER_STATE_STOP,
+
+    /* must be last*/
+    CONTROL_PROTOCOL__HW_INFER_STATE_COUNT
+} CONTROL_PROTOCOL__hw_infer_state_t;
+
+typedef struct {
+    uint32_t hw_infer_state_length;
+    uint8_t hw_infer_state;
+    uint32_t application_index_length;
+    uint8_t application_index;
+    uint32_t dynamic_batch_size_length;
+    uint16_t dynamic_batch_size;
+    uint32_t channels_info_length;
+    CONTROL_PROTOCOL__hw_infer_channels_info_t channels_info;
+} CONTROL_PROTOCOL__change_hw_infer_status_request_t;
+
 typedef union {
     CONTROL_PROTOCOL_identify_response_t identity_response;
     CONTROL_PROTOCOL__core_identify_response_t core_identity_response;
@@ -1305,6 +1345,7 @@ typedef union {
     CONTROL_PROTOCOL__get_throttling_state_response_t get_throttling_state_response;
     CONTROL_PROTOCOL__get_overcurrent_state_response_t get_overcurrent_state_response;
     CONTROL_PROTOCOL__get_hw_consts_response_t get_hw_consts_response;
+    CONTROL_PROTOCOL__change_hw_infer_status_response_t change_hw_infer_status_response;
 
    // Note: This array is larger than any legal request:
    // * Functions in this module won't write more than CONTROL_PROTOCOL__MAX_CONTROL_LENGTH bytes
@@ -1364,6 +1405,7 @@ typedef union {
    CONTROL_PROTOCOL__sensor_set_i2c_bus_index_t sensor_set_i2c_bus_index;
    CONTROL_PROTOCOL__set_overcurrent_state_request_t set_overcurrent_state_request;
    CONTROL_PROTOCOL__set_sleep_state_request_t set_sleep_state_request;
+   CONTROL_PROTOCOL__change_hw_infer_status_request_t change_hw_infer_status_request;
    // Note: This array is larger than any legal request:
    // * Functions in this module won't write more than CONTROL_PROTOCOL__MAX_CONTROL_LENGTH bytes
    //   when recieving a pointer to CONTROL_PROTOCOL__request_parameters_t.
@@ -1427,7 +1469,6 @@ typedef struct {
     bool is_first_control_per_context;
     bool is_last_control_per_context;
     uint8_t context_type; // CONTROL_PROTOCOL__context_switch_context_type_t
-    uint32_t actions_count;
     uint32_t context_network_data_length;
     uint8_t context_network_data[CONTROL_PROTOCOL__CONTEXT_NETWORK_DATA_SINGLE_CONTROL_MAX_SIZE];
 } CONTROL_PROTOCOL__context_switch_context_info_single_control_t;

common/include/firmware_header.h

@@ -18,7 +18,7 @@ extern "C" {
 #include "utils.h"
 
 #define FIRMWARE_HEADER_MAGIC_HAILO8 (0x1DD89DE0)
-#define FIRMWARE_HEADER_MAGIC_MERCURY (0xE905DAAB)
+#define FIRMWARE_HEADER_MAGIC_HAILO15 (0xE905DAAB)
 
 typedef enum {
     FIRMWARE_HEADER_VERSION_INITIAL = 0,
@@ -29,12 +29,12 @@ typedef enum {
 
 typedef enum {
     FIRMWARE_TYPE_HAILO8 = 0,
-    FIRMWARE_TYPE_MERCURY
+    FIRMWARE_TYPE_HAILO15
 } firmware_type_t;
 
 
 #ifdef MERCURY
-#define COMPILED_FIRMWARE_TYPE (FIRMWARE_TYPE_MERCURY)
+#define COMPILED_FIRMWARE_TYPE (FIRMWARE_TYPE_HAILO15)
 #elif defined(HAILO8_B0)
 #define COMPILED_FIRMWARE_TYPE (FIRMWARE_TYPE_HAILO8)
 #endif /* MERCURY */

common/include/firmware_status.h

@@ -409,6 +409,8 @@ Updating rules:
    FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_SET_SLEEP_STATE_FAILED)\
    FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_SLEEP_STATE_SIZE)\
    FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_SLEEP_STATE)\
+   FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_HW_INFER_STATE_LENGTH)\
+   FIRMWARE_STATUS__X(CONTROL_PROTOCOL_STATUS_INVALID_CHANNELS_INFO_LENGTH)\
    \
    FIRMWARE_MODULE__X(FIRMWARE_MODULE__POWER_MEASUREMENT)\
    FIRMWARE_STATUS__X(HAILO_POWER_MEASUREMENT_STATUS_POWER_INIT_ERROR)\
@@ -968,6 +970,8 @@ Updating rules:
    FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_CSM_BURST_COUNTER_IS_NOT_ZERO)\
    FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_CSM_CREDIT_COUNTER_IS_NOT_ZERO)\
    FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_CSM_FIFO_NOT_EMPTY)\
+   FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_INVALID_HOST_PAGE_SIZE)\
+   FIRMWARE_STATUS__X(CSM_CONFIG_MANAGER_STATUS_INVALID_BUFFER_SIZE)\
    \
    FIRMWARE_MODULE__X(FIRMWARE_MODULE__PCIE_CONFIG_MANAGER)\
    FIRMWARE_STATUS__X(PCIE_CONFIG_MANAGER_STATUS_NOT_IMPLEMENTED)\
@@ -1074,6 +1078,7 @@ Updating rules:
    FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_NOT_SUPPORTED_OPERATION)\
    FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_INVALID_NETWORK_INDEX)\
    FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_INVALID_NMS_UNIT_INDEX)\
+   FIRMWARE_STATUS__X(NMS_MANAGER_STATUS_INVALID_BATCH_SIZE)\
    \
    FIRMWARE_MODULE__X(FIRMWARE_MODULE__CLUSTER_MANAGER)\
    FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_CLUSTER_INDEX)\
@@ -1082,6 +1087,12 @@ Updating rules:
    FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_LCU_INDEX)\
    FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_INVALID_KERNEL_DONE_ADDRESS)\
    FIRMWARE_STATUS__X(CLUSTER_MANAGER_STATUS_RECEIVED_UNEXPECTED_INTERRUPT)\
+   \
+   FIRMWARE_MODULE__X(FIRMWARE_MODULE__HW_INFER_MANAGER)\
+   FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_NETWORK_GROUP_NOT_CONFIGURED_BEFORE_INFER_START)\
+   FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_NETWORK_GROUP_ALREADY_ACTIVATED)\
+   FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_STATE_MACHINE_NOT_IN_RESET_STATE_BEFORE_DEACTIVATE)\
+   FIRMWARE_STATUS__X(HW_INFER_MANAGER_STATUS_INVALID_STATE)\
 
 
 typedef enum {

common/src/firmware_header_utils.c

@@ -47,8 +47,8 @@ static HAILO_COMMON_STATUS_t firmware_header_utils__validate_fw_header(uintptr_t
     case FIRMWARE_TYPE_HAILO8:
         firmware_magic = FIRMWARE_HEADER_MAGIC_HAILO8; 
         break;
-    case FIRMWARE_TYPE_MERCURY:
+    case FIRMWARE_TYPE_HAILO15:
-        firmware_magic = FIRMWARE_HEADER_MAGIC_MERCURY; 
+        firmware_magic = FIRMWARE_HEADER_MAGIC_HAILO15;
         break;
     default:
         status = HAILO_STATUS__FIRMWARE_HEADER_UTILS__INVALID_FIRMWARE_TYPE;

hailort/CMakeLists.txt

@@ -10,10 +10,6 @@ option(HAILO_OFFLINE_COMPILATION "Don't download external dependencies" OFF)
 option(HAILO_BUILD_SERVICE "Build hailort service" OFF)
 option(HAILO_BUILD_PROFILER "Build hailort profiler" ON)
 
-if(WIN32 AND ${HAILO_BUILD_SERVICE})
-    message(FATAL_ERROR "HailoRT service is not supported on Windows")
-endif()
-
 # Flag for emulator (FPGA/Veloce)
 if(HAILO_BUILD_EMULATOR)
     message(WARNING "HailoRT is building with Emulator flag on")
@@ -22,8 +18,8 @@ endif()
 
 # Set firmware version
 add_definitions( -DFIRMWARE_VERSION_MAJOR=4 )
-add_definitions( -DFIRMWARE_VERSION_MINOR=12 )
+add_definitions( -DFIRMWARE_VERSION_MINOR=13 )
-add_definitions( -DFIRMWARE_VERSION_REVISION=1 )
+add_definitions( -DFIRMWARE_VERSION_REVISION=0 )
 if(HAILO_BUILD_SERVICE)
     add_definitions( -DHAILO_SUPPORT_MULTI_PROCESS )
 endif()
@@ -86,7 +82,7 @@ if(HAILO_BUILD_PYBIND)
     if(NOT PYTHON_EXECUTABLE AND PYBIND11_PYTHON_VERSION)
         # PYBIND11_PYTHON_VERSION is prioritized (not virtual environment) if PYTHON_EXECUTABLE is not set.
         # See https://pybind11.readthedocs.io/en/stable/changelog.html#v2-6-0-oct-21-2020
-        if(${CMAKE_VERSION} VERSION_LESS "3.22.0")
+        if((${CMAKE_VERSION} VERSION_LESS "3.22.0") AND (NOT WIN32))
             find_package(PythonInterp ${PYBIND11_PYTHON_VERSION} REQUIRED)
             set(PYTHON_EXECUTABLE ${Python_EXECUTABLE})
         else()

hailort/common/CMakeLists.txt

@@ -14,10 +14,13 @@ set(SRC_FILES
     ${HAILORT_COMMON_OS_DIR}/filesystem.cpp
     ${HAILORT_COMMON_OS_DIR}/socket.cpp
     ${HAILORT_COMMON_OS_DIR}/process.cpp
+    ${HAILORT_COMMON_OS_DIR}/os_utils.cpp
 
     ${CMAKE_CURRENT_SOURCE_DIR}/barrier.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/file_utils.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/string_utils.cpp
+
+    ${CMAKE_CURRENT_SOURCE_DIR}/device_measurements.cpp
 )
 
 if(WIN32)

hailort/common/async_thread.hpp

@@ -13,6 +13,8 @@
 #include <thread>
 #include <memory>
 
+#include "common/os_utils.hpp"
+
 namespace hailort
 {
 
@@ -23,16 +25,29 @@ namespace hailort
 template<typename T>
 class AsyncThread final {
 public:
-    explicit AsyncThread(std::function<T(void)> func) :
+    AsyncThread(const std::string &name, std::function<T(void)> func) :
         m_result(),
-        m_thread([this, func]() {
+        m_thread([this, name, func]() {
+            if (!name.empty()) {
+                OsUtils::set_current_thread_name(name);
+            }
             m_result = func();
         })
     {}
 
+    explicit AsyncThread(std::function<T(void)> func) : AsyncThread("", func)
+    {}
+
+    ~AsyncThread()
+    {
+        // Join on the thread. this can be a blocking operation, so to avoid it the user must call .get()
+        // before the object gets destracted (same behavoiur as in std::future returned from std::async).
+        get();
+    }
+
     /**
      * NOTE! this object is not moveable by purpose, on creation we create a lambda that take `this`, if we
-     * move the object `this` will change and the callback will be wrong. Use exeternal storage like std::unique_ptr
+     * move the object `this` will change and the callback will be wrong. Use external storage like std::unique_ptr
      * to move the object (or to put it inside a container)
      */
     AsyncThread(const AsyncThread<T> &) = delete;

hailort/common/circular_buffer.hpp

@@ -15,6 +15,7 @@
 #include "hailo/platform.h"
 #include "common/utils.hpp"
 #include <array>
+#include <iterator>
 
 namespace hailort
 {
@@ -60,14 +61,43 @@ typedef struct {
     MIN(CB_PROG((circbuf), (head), (tail)), (circbuf).size - (tail))
 
 
+template<typename T>
+struct is_std_array : public std::false_type {};
+
+template<typename T, std::size_t N>
+struct is_std_array<std::array<T, N>> : public std::true_type {};
+
 // TODO: implement more functionalities, better move semantic handle
 // TODO: support consts methods (front(), empty()), right now CB_* macros requires non const pointer to head+tail
-template<typename T>
+template<typename T, typename Container = std::vector<T>>
 class CircularArray final
 {
 public:
-    static_assert(std::is_pod<T>::value, "CircularArray can be used only with POD type");
 
+    static_assert(std::is_default_constructible<T>::value, "CircularArray object must be default constructible");
+
+    // Based on https://en.cppreference.com/w/cpp/iterator/iterator
+    class iterator: public std::iterator<std::input_iterator_tag,   // iterator_category
+                                         T,                         // value_type
+                                         int,                       // difference_type
+                                         int,                       // pointer
+                                         T&>                        // reference
+    {
+    public:
+        explicit iterator(int index, CircularArray &array) : m_array(array), m_index(index) {}
+        iterator& operator++() {  m_index = ((m_index + 1) & m_array.m_circ.size_mask); return *this; }
+        iterator operator++(int) { iterator retval = *this; ++(*this); return retval; }
+        bool operator==(iterator other) const { return m_index == other.m_index; }
+        bool operator!=(iterator other) const { return !(*this == other); }
+        T &operator*() const { return m_array.m_array[m_index]; }
+    private:
+        CircularArray &m_array;
+        int m_index;
+    };
+
+    // Ctor for Container=std::vector
+    template <typename C=Container,
+        class = typename std::enable_if_t<std::is_same<C, std::vector<T>>::value>>
     CircularArray(size_t storage_size)
     {
         // storage size must be a power of 2
@@ -76,23 +106,55 @@ public:
         m_array.resize(storage_size);
     }
 
+    // Ctor for Container=std::array
+    template <typename C=Container,
+        class = typename std::enable_if_t<is_std_array<C>::value>>
+    CircularArray(size_t storage_size, int = 0)
+    {
+        // storage size must be a power of 2
+        assert(is_powerof2(storage_size));
+        assert(storage_size <= std::tuple_size<C>::value);
+        CB_INIT(m_circ, storage_size);
+    }
+
+    void push_back(T &&element)
+    {
+        assert(!full());
+        m_array[CB_HEAD(m_circ)] = std::move(element);
+        CB_ENQUEUE(m_circ, 1);
+    }
+
     void push_back(const T& element)
     {
-        assert(CB_AVAIL(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ)));
+        assert(!full());
         m_array[CB_HEAD(m_circ)] = element;
         CB_ENQUEUE(m_circ, 1);
     }
 
     void pop_front()
     {
+        assert(!empty());
+        // Clear previous front
+        m_array[CB_TAIL(m_circ)] = T();
         CB_DEQUEUE(m_circ, 1);
     }
 
     T &front()
     {
+        assert(!empty());
         return m_array[CB_TAIL(m_circ)];
     }
 
+    void reset()
+    {
+        // pop all fronts to make sure all destructors are called.
+        // TODO: if T is std::is_trivial, we can just reset the counters
+        const auto original_size = size();
+        for (size_t i = 0 ; i < original_size; i++) {
+            pop_front();
+        }
+    }
+
     bool empty()
     {
         return CB_HEAD(m_circ) == CB_TAIL(m_circ);
@@ -108,9 +170,24 @@ public:
         return CB_PROG(m_circ, CB_HEAD(m_circ), CB_TAIL(m_circ));
     }
 
+    size_t capacity()
+    {
+        return CB_SIZE(m_circ) - 1;
+    }
+
+    iterator begin()
+    {
+        return iterator(CB_TAIL(m_circ), *this);
+    }
+
+    iterator end()
+    {
+        return iterator(CB_HEAD(m_circ), *this);
+    }
+
 private:
     circbuf_t m_circ;
-    std::vector<T> m_array;
+    Container m_array;
 };
 
 } /* namespace hailort */

hailort/common/device_measurements.cpp

@@ -0,0 +1,133 @@
+/**
+ * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file device_measurements.hpp
+ * @brief Measure temperature, power and current of Hailo chip
+ **/
+
+
+#include "common/device_measurements.hpp"
+#include "common/utils.hpp"
+
+using namespace hailort;
+
+constexpr std::chrono::milliseconds DEFAULT_MEASUREMENTS_INTERVAL(100);
+
+BaseMeasurement::BaseMeasurement(Device &device, hailo_status &status) :
+    m_device(device),
+    m_is_thread_running(false),
+    m_acc(make_shared_nothrow<FullAccumulator<double>>("BaseMeasurementAccumulator"))
+{
+    if (nullptr == m_acc) {
+        status = HAILO_OUT_OF_HOST_MEMORY;
+        return;
+    }
+    status = HAILO_SUCCESS;
+}
+
+BaseMeasurement::~BaseMeasurement()
+{
+    stop_measurement();
+}
+
+void BaseMeasurement::stop_measurement()
+{
+    m_is_thread_running = false;
+    if (m_thread.joinable()) {
+        m_thread.join();
+    }
+}
+
+AccumulatorResults BaseMeasurement::get_data()
+{
+    std::unique_lock<std::mutex> lock(m_mutex);
+    return m_acc->get();
+}
+
+Expected<std::shared_ptr<TemperatureMeasurement>> TemperatureMeasurement::create_shared(Device &device)
+{
+    auto status = HAILO_UNINITIALIZED;
+    auto ptr = make_shared_nothrow<TemperatureMeasurement>(device, status);
+    CHECK_SUCCESS_AS_EXPECTED(status);
+    CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+    return ptr;
+}
+
+
+TemperatureMeasurement::TemperatureMeasurement(Device &device, hailo_status &status) : BaseMeasurement(device, status)
+{}
+
+hailo_status TemperatureMeasurement::start_measurement()
+{
+    // Checking sensor before starting thread
+    auto temp_info = m_device.get_chip_temperature();
+    CHECK_EXPECTED_AS_STATUS(temp_info);
+
+    m_is_thread_running = true;
+    m_thread = std::thread([this] () {
+        while (m_is_thread_running.load()) {
+            auto temp_info = m_device.get_chip_temperature();
+            if (HAILO_SUCCESS != temp_info.status()) {
+                LOGGER__ERROR("Failed to get chip's temperature, status = {}", temp_info.status());
+                m_is_thread_running = false;
+                break;
+            }
+
+            float32_t ts_avg = ((temp_info->ts0_temperature + temp_info->ts1_temperature) / 2);
+            {
+                std::unique_lock<std::mutex> lock(m_mutex);
+                m_acc->add_data_point(ts_avg, temp_info->sample_count);
+            }
+            
+            std::this_thread::sleep_for(DEFAULT_MEASUREMENTS_INTERVAL); 
+        }
+    });
+
+    return HAILO_SUCCESS;
+}
+
+Expected<std::shared_ptr<PowerMeasurement>> PowerMeasurement::create_shared(Device &device,
+    hailo_power_measurement_types_t measurement_type)
+{
+    auto status = HAILO_UNINITIALIZED;
+    auto ptr = make_shared_nothrow<PowerMeasurement>(device, measurement_type, status);
+    CHECK_SUCCESS_AS_EXPECTED(status);
+    CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+    return ptr;
+}
+
+PowerMeasurement::PowerMeasurement(Device &device, hailo_power_measurement_types_t measurement_type, hailo_status &status)
+    : BaseMeasurement(device, status), m_measurement_type(measurement_type)
+{}
+
+hailo_status PowerMeasurement::start_measurement()
+{
+    // Checking sensor before starting thread
+    auto power_info = m_device.power_measurement(HAILO_DVM_OPTIONS_AUTO, m_measurement_type);
+    CHECK_EXPECTED_AS_STATUS(power_info);
+
+    m_is_thread_running = true;
+    m_thread = std::thread([this] () {
+        while (m_is_thread_running.load()) {
+            auto power_info = m_device.power_measurement(HAILO_DVM_OPTIONS_AUTO, m_measurement_type);
+            if (HAILO_SUCCESS != power_info.status()) {
+                LOGGER__ERROR("Failed to get chip's power, status = {}", power_info.status());
+                m_is_thread_running = false;
+                break;
+            }
+
+            {
+                std::unique_lock<std::mutex> lock(m_mutex);
+                m_acc->add_data_point(*power_info);
+            }
+            
+            std::this_thread::sleep_for(DEFAULT_MEASUREMENTS_INTERVAL); 
+        }
+    });
+
+    return HAILO_SUCCESS;
+}

hailort/common/device_measurements.hpp

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file device_measurements.hpp
+ * @brief Measure temperature, power and current of Hailo chip
+ **/
+
+#ifndef _HAILO_DEVICE_MEASUREMENTS_HPP_
+#define _HAILO_DEVICE_MEASUREMENTS_HPP_
+
+#include "hailo/hailort.h"
+#include "hailo/device.hpp"
+
+#include "common/runtime_statistics_internal.hpp"
+
+#include <thread>
+#include <mutex>
+#include <atomic>
+
+
+class BaseMeasurement
+{
+public:
+    BaseMeasurement(hailort::Device &device, hailo_status &status);
+    virtual ~BaseMeasurement();
+
+    virtual hailo_status start_measurement() = 0;
+    void stop_measurement();
+    hailort::AccumulatorResults get_data();
+
+    virtual std::string measurement_unit() = 0;
+
+protected:
+    hailort::Device &m_device;
+    std::thread m_thread;
+    std::atomic_bool m_is_thread_running;
+    std::mutex m_mutex;
+    hailort::AccumulatorPtr m_acc;
+};
+
+
+class TemperatureMeasurement : public BaseMeasurement
+{
+public:
+    static hailort::Expected<std::shared_ptr<TemperatureMeasurement>> create_shared(hailort::Device &device);
+
+    virtual ~TemperatureMeasurement() = default;
+
+    virtual hailo_status start_measurement() override;
+
+    virtual std::string measurement_unit() override
+    {
+        return "C";
+    }
+
+    TemperatureMeasurement(hailort::Device &device, hailo_status &status);
+};
+
+
+class PowerMeasurement : public BaseMeasurement
+{
+public:
+    static hailort::Expected<std::shared_ptr<PowerMeasurement>> create_shared(hailort::Device &device,
+        hailo_power_measurement_types_t measurement_type);
+    virtual ~PowerMeasurement() = default;
+
+    virtual hailo_status start_measurement() override;
+
+    virtual std::string measurement_unit() override
+    {
+        switch (m_measurement_type) {
+        case HAILO_POWER_MEASUREMENT_TYPES__SHUNT_VOLTAGE:
+        case HAILO_POWER_MEASUREMENT_TYPES__BUS_VOLTAGE:
+            return "mV";
+        case HAILO_POWER_MEASUREMENT_TYPES__AUTO:
+        case HAILO_POWER_MEASUREMENT_TYPES__POWER:
+            return "W";
+        case HAILO_POWER_MEASUREMENT_TYPES__CURRENT:
+            return "mA";
+        default:
+            return "Nan";
+        };
+    }
+
+    PowerMeasurement(hailort::Device &device, hailo_power_measurement_types_t measurement_type,
+        hailo_status &status);
+
+private:
+    hailo_power_measurement_types_t m_measurement_type;
+};
+
+#endif /* _HAILO_DEVICE_MEASUREMENTS_HPP_ */

hailort/common/ethernet_utils.hpp

@@ -12,7 +12,7 @@
 #ifndef __OS_ETHERNET_UTILS_H__
 #define __OS_ETHERNET_UTILS_H__
 
-
+#include <string>
 #include <hailo/hailort.h>
 #include "hailo/expected.hpp"
 

hailort/common/latency_meter.hpp

@@ -93,7 +93,6 @@ private:
             return;
         }
 
-        duration start = m_start_timestamps.front();
         duration end(0);
         for (auto &end_timesatmps : m_end_timestamps_per_channel) {
             if (end_timesatmps.second.empty()) {
@@ -104,6 +103,9 @@ private:
             end = std::max(end, end_timesatmps.second.front());
         }
 
+        duration start = m_start_timestamps.front();
+        assert(start <= end);
+
         // calculate the latency
         m_latency_sum += (end - start);
         m_latency_count++;

hailort/common/os/posix/os_utils.cpp

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file os_utils.cpp
+ * @brief Utilities for Posix methods
+ **/
+
+#include "hailo/hailort.h"
+
+#include "common/os_utils.hpp"
+
+#include "spdlog/sinks/syslog_sink.h"
+
+
+namespace hailort
+{
+
+HailoRTOSLogger::HailoRTOSLogger()
+{
+    m_hailort_os_logger = spdlog::syslog_logger_mt("syslog", "hailort_service", LOG_PID);
+    m_hailort_os_logger->set_pattern("%v");
+    m_hailort_os_logger->set_level(spdlog::level::debug);
+}
+
+uint32_t OsUtils::get_curr_pid()
+{
+    return getpid();
+}
+
+CursorAdjustment::CursorAdjustment(){}
+CursorAdjustment::~CursorAdjustment(){}
+
+} /* namespace hailort */

hailort/common/os/windows/os_utils.cpp

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file os_utils.cpp
+ * @brief Utilities for Windows methods
+ **/
+
+#include "common/os_utils.hpp"
+#include "hailo/hailort.h"
+
+#include <windows.h>
+#include "spdlog/sinks/win_eventlog_sink.h"
+
+namespace hailort
+{
+
+HailoRTOSLogger::HailoRTOSLogger()
+{
+    auto event_log_sink = std::make_shared<spdlog::sinks::win_eventlog_sink_mt>("hailort_service");
+    m_hailort_os_logger = std::make_shared<spdlog::logger>("eventlog", event_log_sink);
+    event_log_sink->set_pattern("%v");
+    m_hailort_os_logger->set_level(spdlog::level::debug);
+}
+
+uint32_t OsUtils::get_curr_pid()
+{
+    return static_cast<uint32_t>(GetCurrentProcessId());
+}
+
+CursorAdjustment::CursorAdjustment()
+{
+    // Enables Vitual Terminal Processing - enables ANSI Escape Sequences on Windows
+    // Source: https://stackoverflow.com/questions/52607960/how-can-i-enable-virtual-terminal-processing
+    HANDLE h_out = GetStdHandle(STD_OUTPUT_HANDLE);
+    DWORD dword_mode = 0;
+    GetConsoleMode(h_out, &dword_mode);
+    m_previous_output_buffer_mode = dword_mode;
+    dword_mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+    SetConsoleMode(h_out, dword_mode);
+}
+
+CursorAdjustment::~CursorAdjustment()
+{
+        // Return to the original state
+        HANDLE h_out = GetStdHandle(STD_OUTPUT_HANDLE);
+        SetConsoleMode(h_out, m_previous_output_buffer_mode); // Return the output buffer mode to it's original mode
+}
+} /* namespace hailort */

hailort/common/os_utils.hpp

@@ -0,0 +1,80 @@
+/**
+ * Copyright (c) 2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+**/
+/**
+ * @file os_utils.hpp
+ * @brief Utilities for OS methods
+ **/
+
+#ifndef _HAILO_OS_UTILS_HPP_
+#define _HAILO_OS_UTILS_HPP_
+
+#include "hailo/hailort.h"
+
+#include "common/logger_macros.hpp"
+
+
+namespace hailort
+{
+
+class HailoRTOSLogger final
+{
+public:
+    static HailoRTOSLogger& get_instance()
+    {
+        static HailoRTOSLogger instance;
+        return instance;
+    }
+
+    std::shared_ptr<spdlog::logger> logger()
+    {
+        return m_hailort_os_logger;
+    }
+
+private:
+    HailoRTOSLogger();
+    std::shared_ptr<spdlog::logger> m_hailort_os_logger;
+};
+
+class CursorAdjustment final
+{
+public:
+    CursorAdjustment();
+    ~CursorAdjustment();
+private:
+#if defined(_WIN32)
+    unsigned int m_previous_output_buffer_mode;
+#endif /* _WIN32 */
+};
+
+#define _HAILORT_OS_LOG(level, ...)  SPDLOG_LOGGER_CALL(hailort::HailoRTOSLogger::get_instance().logger(), level, __VA_ARGS__)
+#define HAILORT_OS_LOG_INFO(...)  _HAILORT_OS_LOG(spdlog::level::info, __VA_ARGS__)
+#define HAILORT_OS_LOG_WARNNING(...)  _HAILORT_OS_LOG(spdlog::level::warn, __VA_ARGS__)
+#define HAILORT_OS_LOG_ERROR(...)  _HAILORT_OS_LOG(spdlog::level::err, __VA_ARGS__)
+
+class OsUtils final
+{
+public:
+    OsUtils() = delete;
+    static uint32_t get_curr_pid();
+
+    static void set_current_thread_name(const std::string &name)
+    {
+        (void)name;
+#ifndef NDEBUG
+#ifndef _WIN32
+    // pthread_setname_np name size is limited to 16 chars (including null terminator)
+    assert(name.size() < 16);
+    pthread_setname_np(pthread_self(), name.c_str());
+#else
+// TODO: implement for windows
+#endif /* _WIN32 */
+#endif /* NDEBUG */
+    }
+
+};
+
+} /* namespace hailort */
+
+#endif /* _HAILO_OS_UTILS_HPP_ */

hailort/common/runtime_statistics_internal.hpp

@@ -39,17 +39,17 @@ public:
     FullAccumulator &operator=(const FullAccumulator &) = delete;
     virtual ~FullAccumulator() = default;
 
-    virtual void add_data_point(T data) override
+    virtual void add_data_point(T data, uint32_t samples_count = 1) override
     {
         std::lock_guard<std::recursive_mutex> lock_guard(m_lock);
         m_min = std::min(m_min, static_cast<double>(data));
         m_max = std::max(m_max, static_cast<double>(data));
-        m_count++;
+        m_count += samples_count;
         
         // mean, variance, sd and mean_sd are calculated using Welford's_online_algorithm.
         // See: https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
         const auto delta = static_cast<double>(data) - m_mean;
-        m_mean += delta / static_cast<double>(m_count);
+        m_mean += ((delta * samples_count )/ static_cast<double>(m_count));
         m_M2 += delta * (static_cast<double>(data) - m_mean);
     }
 
@@ -182,7 +182,7 @@ public:
 
     // data is a duration of time.
     // However, the statistics collected will be in frames per seconds (i.e. time^-1).
-    virtual void add_data_point(T data) override
+    virtual void add_data_point(T data, uint32_t samples_count = 1) override
     {
         assert(0 != data);
 
@@ -192,13 +192,13 @@ public:
         // Note: 'this' is needed to access protected members of a template base class
         this->m_min = std::min(this->m_min, data_inverse);
         this->m_max = std::max(this->m_max, data_inverse);
-        this->m_count++;
+        this->m_count += samples_count;
         
         // mean, variance, sd and mean_sd are calculated using Welford's_online_algorithm.
         // See: https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
         const auto delta = data_inverse - this->m_mean;
         // We calculate the arithmatic mean
-        this->m_mean = static_cast<double>(this->m_count) / static_cast<double>(m_sum);
+        this->m_mean = static_cast<double>(this->m_count * samples_count) / static_cast<double>(m_sum);
         this->m_M2 += delta * (data_inverse - this->m_mean);
     }
 

hailort/common/utils.hpp

@@ -206,13 +206,12 @@ _ISEMPTY(                                                               \
 #define _CHECK_SUCCESS_AS_RPC_STATUS(status, reply, is_default, fmt, ...)                                                                       \
     do {                                                                                                                                        \
         const auto &__check_success_status = (status);                                                                                          \
-        if (__check_success_status != HAILO_SUCCESS) {                                                                                       \
         reply->set_status(static_cast<uint32_t>(__check_success_status));                                                                       \
-            LOGGER__ERROR(                                                                                                                   \
+        _CHECK(                                                                                                                                 \
+            HAILO_SUCCESS == __check_success_status,                                                                                            \
+            grpc::Status::OK,                                                                                                                   \
             _CONSTRUCT_MSG(is_default, "CHECK_SUCCESS_AS_RPC_STATUS failed with status={}", fmt, __check_success_status, ##__VA_ARGS__)         \
         );                                                                                                                                      \
-            return grpc::Status::OK;                                                                                                         \
-        }                                                                                                                                    \
     } while(0)
 #define CHECK_SUCCESS_AS_RPC_STATUS(status, reply, ...) _CHECK_SUCCESS_AS_RPC_STATUS(status, reply, ISEMPTY(__VA_ARGS__), "" __VA_ARGS__)
 

hailort/drivers/common/hailo_ioctl_common.h

@@ -7,8 +7,6 @@
 #define _HAILO_IOCTL_COMMON_H_
 
 
-#define DESCRIPTORS_IN_BUFFER(buffer_size, desc_page_size) (((buffer_size) + (desc_page_size) - 1) / (desc_page_size))
-
 // This value is not easily changeable.
 // For example: the channel interrupts ioctls assume we have up to 32 channels
 #define MAX_VDMA_CHANNELS_PER_ENGINE    (32)
@@ -19,7 +17,6 @@
 #define CHANNEL_IRQ_TIMESTAMPS_SIZE (128 * 2) // Should be same as MAX_IRQ_TIMESTAMPS_SIZE (hailort_driver.hpp)
 #define CHANNEL_IRQ_TIMESTAMPS_SIZE_MASK (CHANNEL_IRQ_TIMESTAMPS_SIZE - 1)
 
-#define INVALID_CHANNEL_HANDLE_VALUE    ((uint64_t)-1)
 #define INVALID_DRIVER_HANDLE_VALUE     ((uintptr_t)-1)
 
 // Used by windows and unix driver to raise the right CPU control handle to the FW. The same as in pcie_service FW 
@@ -153,54 +150,43 @@ struct hailo_desc_list_bind_vdma_buffer_params {
     uintptr_t desc_handle;      // in
     uint16_t desc_page_size;    // in
     uint8_t channel_index;      // in
-    size_t offset;              // in
+    uint32_t starting_desc;     // in
 };
 
-/* structure used in ioctl HAILO_VDMA_CHANNEL_ENABLE */
+/* structure used in ioctl HAILO_VDMA_INTERRUPTS_ENABLE */
-struct hailo_vdma_channel_enable_params {
+struct hailo_vdma_interrupts_enable_params {
-    uint8_t engine_index;                       // in
+    uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES];  // in
-    uint8_t channel_index;                      // in
-    enum hailo_dma_data_direction direction;    // in
     bool enable_timestamps_measure;                         // in
-    uint64_t channel_handle;                    // out
 };
 
-/* structure used in ioctl HAILO_VDMA_CHANNEL_DISABLE */
+/* structure used in ioctl HAILO_VDMA_INTERRUPTS_DISABLE */
-struct hailo_vdma_channel_disable_params {
+struct hailo_vdma_interrupts_disable_params {
-    uint8_t engine_index;    // in
+    uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES];  // in
-    uint8_t channel_index;   // in
-    uint64_t channel_handle; // in
 };
 
-/* structure used in ioctl HAILO_VDMA_CHANNEL_WAIT_INT */
+/* structure used in ioctl HAILO_VDMA_INTERRUPTS_WAIT */
-struct hailo_vdma_channel_wait_params {
+struct hailo_vdma_interrupts_channel_data {
+    uint8_t engine_index;
+    uint8_t channel_index;
+    bool is_active;                 // If not activate, num_processed is ignored.
+    uint16_t host_num_processed;
+    uint8_t host_error;             // Channel errors bits on source side
+    uint8_t device_error;           // Channel errors bits on dest side
+};
+
+struct hailo_vdma_interrupts_wait_params {
+    uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES];          // in
+    uint8_t channels_count;                                         // out
+    struct hailo_vdma_interrupts_channel_data
+        irq_data[MAX_VDMA_CHANNELS_PER_ENGINE * MAX_VDMA_ENGINES];  // out
+};
+
+/* structure used in ioctl HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS */
+struct hailo_vdma_interrupts_read_timestamp_params {
     uint8_t engine_index;                                                               // in
     uint8_t channel_index;                                                              // in
-    uint64_t channel_handle;                                                            // in
+    uint32_t timestamps_count;                                                          // out
-    uint64_t timeout_ms;                                                                // in
-    uint32_t timestamps_count;                                                          // inout
-// In linux send address to local buffer because there isnt room on stack for array
-#if defined(__linux__)
-    struct hailo_channel_interrupt_timestamp *timestamps;                               // out
-#elif defined(__QNX__) || defined(_MSC_VER)
     struct hailo_channel_interrupt_timestamp timestamps[CHANNEL_IRQ_TIMESTAMPS_SIZE];   // out
-#else
-#error "unsupported platform!"
-#endif // __linux__  
-};
-
-/* structure used in ioctl HAILO_VDMA_CHANNEL_ABORT */
-struct hailo_vdma_channel_abort_params {
-    uint8_t engine_index;       // in
-    uint8_t channel_index;      // in
-    uint64_t channel_handle;    // in
-};
-
-/* structure used in ioctl HAILO_VDMA_CHANNEL_CLEAR_ABORT */
-struct hailo_vdma_channel_clear_abort_params {
-    uint8_t engine_index;       // in
-    uint8_t channel_index;      // in
-    uint64_t channel_handle;    // in
 };
 
 /* structure used in ioctl HAILO_FW_CONTROL */
@@ -301,8 +287,8 @@ enum hailo_vdma_buffer_sync_type {
 struct hailo_vdma_buffer_sync_params {
     size_t handle;                              // in
     enum hailo_vdma_buffer_sync_type sync_type; // in
-    void*                             buffer_address;   // in
+    size_t offset;                              // in
-    uint64_t                          buffer_size;      // in
+    size_t count;                               // in
 };
 
 /* structure used in ioctl HAILO_READ_NOTIFICATION */
@@ -315,7 +301,7 @@ struct hailo_d2h_notification {
 
 enum hailo_board_type {
     HAILO_BOARD_TYPE_HAILO8 = 0,
-    HAILO_BOARD_TYPE_MERCURY,
+    HAILO_BOARD_TYPE_HAILO15,
     HAILO_BOARD_TYPE_COUNT,
 
     /** Max enum value to maintain ABI Integrity */
@@ -399,11 +385,10 @@ enum hailo_general_ioctl_code {
 #define HAILO_RESET_NN_CORE             _IO_(HAILO_GENERAL_IOCTL_MAGIC,    HAILO_RESET_NN_CORE_CODE)
 
 enum hailo_vdma_ioctl_code {
-    HAILO_VDMA_CHANNEL_ENABLE_CODE,
+    HAILO_VDMA_INTERRUPTS_ENABLE_CODE,
-    HAILO_VDMA_CHANNEL_DISABLE_CODE,
+    HAILO_VDMA_INTERRUPTS_DISABLE_CODE,
-    HAILO_VDMA_CHANNEL_WAIT_INT_CODE,
+    HAILO_VDMA_INTERRUPTS_WAIT_CODE,
-    HAILO_VDMA_CHANNEL_ABORT_CODE,
+    HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE,
-    HAILO_VDMA_CHANNEL_CLEAR_ABORT_CODE,
     HAILO_VDMA_CHANNEL_READ_REGISTER_CODE,
     HAILO_VDMA_CHANNEL_WRITE_REGISTER_CODE,
     HAILO_VDMA_BUFFER_MAP_CODE,
@@ -422,11 +407,11 @@ enum hailo_vdma_ioctl_code {
     HAILO_VDMA_IOCTL_MAX_NR,
 };
 
-#define HAILO_VDMA_CHANNEL_ENABLE           _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_ENABLE_CODE,         struct hailo_vdma_channel_enable_params)
+#define HAILO_VDMA_INTERRUPTS_ENABLE          _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_INTERRUPTS_ENABLE_CODE,            struct hailo_vdma_interrupts_enable_params)
-#define HAILO_VDMA_CHANNEL_DISABLE          _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_CHANNEL_DISABLE_CODE,        struct hailo_vdma_channel_disable_params)
+#define HAILO_VDMA_INTERRUPTS_DISABLE         _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_INTERRUPTS_DISABLE_CODE,           struct hailo_vdma_interrupts_disable_params)
-#define HAILO_VDMA_CHANNEL_WAIT_INT         _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_WAIT_INT_CODE,       struct hailo_vdma_channel_wait_params)
+#define HAILO_VDMA_INTERRUPTS_WAIT            _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_WAIT_CODE,              struct hailo_vdma_interrupts_wait_params)
-#define HAILO_VDMA_CHANNEL_ABORT            _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_CHANNEL_ABORT_CODE,          struct hailo_vdma_channel_abort_params)
+#define HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE,   struct hailo_vdma_interrupts_read_timestamp_params)
-#define HAILO_VDMA_CHANNEL_CLEAR_ABORT      _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_CHANNEL_CLEAR_ABORT_CODE,    struct hailo_vdma_channel_clear_abort_params)
+
 #define HAILO_VDMA_CHANNEL_READ_REGISTER      _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CHANNEL_READ_REGISTER_CODE,         struct hailo_vdma_channel_read_register_params)
 #define HAILO_VDMA_CHANNEL_WRITE_REGISTER     _IOR_(HAILO_VDMA_IOCTL_MAGIC,  HAILO_VDMA_CHANNEL_WRITE_REGISTER_CODE,        struct hailo_vdma_channel_write_register_params)
 

hailort/drivers/win/include/Public.h

@@ -56,7 +56,6 @@ struct tCommonHailoIoctlParam
 #define HAILO_CMD_MAP_BUFFER            0x0051
 #define HAILO_CMD_FREE_MEMORY           0x0060
 #define HAILO_CMD_ALLOC_MEMORY          0x0061
-#define HAILO_CMD_ABORT_ALL             0x0070
 
 #define HAILO_IOCTL_COMPATIBLE                  CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_ANY_ACCESS)
 struct tCompatibleHailoIoctlParam
@@ -103,11 +102,10 @@ struct tCompatibleHailoIoctlData
     ULONG_PTR Value;
     union {
         hailo_memory_transfer_params MemoryTransfer;
-        hailo_vdma_channel_enable_params ChannelEnable;
+        hailo_vdma_interrupts_enable_params VdmaInterruptsEnable;
-        hailo_vdma_channel_disable_params ChannelDisable;
+        hailo_vdma_interrupts_disable_params VdmaInterruptsDisable;
-        hailo_vdma_channel_wait_params ChannelWait;
+        hailo_vdma_interrupts_read_timestamp_params VdmaInterruptsReadTimestamps;
-        hailo_vdma_channel_abort_params ChannelAbort;
+        hailo_vdma_interrupts_wait_params VdmaInterruptsWait;
-        hailo_vdma_channel_clear_abort_params ChannelClearAbort;
         hailo_vdma_buffer_sync_params VdmaBufferSync;
         hailo_fw_control FirmwareControl;
         hailo_vdma_buffer_map_params VdmaBufferMap;

hailort/hailort_service/CMakeLists.txt

@@ -1,9 +1,17 @@
 cmake_minimum_required(VERSION 3.0.0)
 
+if(WIN32)
+    set(HAILORT_SERVICE_OS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/windows")
+elseif(UNIX)
+    set(HAILORT_SERVICE_OS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/unix")
+else()
+    message(FATAL_ERROR "Unexpeced platform target, stopping build")
+endif()
+
 add_executable(hailort_service
     hailort_rpc_service.cpp
-    hailort_service.cpp
     service_resource_manager.hpp
+    ${HAILORT_SERVICE_OS_DIR}/hailort_service.cpp
     ${HAILORT_COMMON_CPP_SOURCES}
 )
 target_compile_options(hailort_service PRIVATE ${HAILORT_COMPILE_OPTIONS})
@@ -12,9 +20,16 @@ target_link_libraries(hailort_service
     libhailort
     spdlog::spdlog
     grpc++_unsecure
-    hailort_rpc_grpc_proto)
+    hailort_rpc_grpc_proto
+)
+if(WIN32)
+    # Needed in order to compile eth utils (we compile here ${HAILORT_COMMON_CPP_SOURCES}, consider removing)
+    target_link_libraries(hailort_service Iphlpapi Shlwapi Kernel32 Advapi32)
+endif()
+
 target_include_directories(hailort_service
     PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
     ${HAILORT_INC_DIR}
     ${HAILORT_COMMON_DIR}
     ${COMMON_INC_DIR}
@@ -27,24 +42,26 @@ if(NOT CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
     set(SYSTEMD_UNIT_DIR ${CMAKE_INSTALL_PREFIX}/${SYSTEMD_UNIT_DIR})
 endif()
 
-# Install service's environment variables file
+if (UNIX)
-set(ENV_VARS_FILE_DIR ${CMAKE_INSTALL_SYSCONFDIR}/default/)
+    # Install service's environment variables file
-set(HAILORT_SERVICE_ENV_VARS_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort_service)
+    set(ENV_VARS_FILE_DIR ${CMAKE_INSTALL_SYSCONFDIR}/default/)
-install(
+    set(HAILORT_SERVICE_ENV_VARS_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort_service)
+    install(
         FILES "${HAILORT_SERVICE_ENV_VARS_FILE}"
         DESTINATION "${ENV_VARS_FILE_DIR}"
         CONFIGURATIONS Release
         COMPONENT hailort_service
-)
+    )
 
-# Install systemd unit file
+    # Install systemd unit file
-set(HAILORT_SERVICE_UNIT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort.service)
+    set(HAILORT_SERVICE_UNIT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hailort.service)
-install(
+    install(
         FILES "${HAILORT_SERVICE_UNIT_FILE}"
         DESTINATION "${SYSTEMD_UNIT_DIR}"
         CONFIGURATIONS Release
         COMPONENT hailort_service
-)
+    )
+endif()
 
 install(
     TARGETS hailort_service

hailort/hailort_service/hailort_rpc_service.cpp

@@ -7,32 +7,63 @@
  * @brief Implementation of the hailort rpc service
  **/
 
-#include "hailort_rpc_service.hpp"
-#include "rpc/rpc_definitions.hpp"
-#include "service_resource_manager.hpp"
-#include "common/utils.hpp"
 #include "hailo/network_group.hpp"
 #include "hailo/vdevice.hpp"
 #include "hailo/vstream.hpp"
 #include "hailo/hailort_common.hpp"
-#include <syslog.h>
+
+#include "common/utils.hpp"
+#include "common/os_utils.hpp"
+
+#include "hailort_rpc_service.hpp"
+#include "rpc/rpc_definitions.hpp"
+#include "service_resource_manager.hpp"
+
+#include <thread>
 
 namespace hailort
 {
 
-grpc::Status HailoRtRpcService::client_keep_alive(grpc::ServerContext *ctx, const keepalive_Request *request,
+HailoRtRpcService::HailoRtRpcService()
-    empty*)
+    : ProtoHailoRtRpc::Service()
 {
-    auto client_id = request->process_id();
+    m_keep_alive = make_unique_nothrow<std::thread>([this] () {
-    while (!ctx->IsCancelled()) {
+        this->keep_alive();
-        sleep(hailort::HAILO_KEEPALIVE_INTERVAL_SEC);
+    });
-    }
+}
+
+void HailoRtRpcService::keep_alive()
+{
+    while (true) {
+        std::this_thread::sleep_for(hailort::HAILO_KEEPALIVE_INTERVAL / 2);
+        auto now = std::chrono::high_resolution_clock::now();
+        std::unique_lock<std::mutex> lock(m_mutex);
+        std::set<uint32_t> pids_to_remove;
+        for (auto pid_to_last_alive : m_clients_pids) {
+            auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - pid_to_last_alive.second);
+            if (duration > hailort::HAILO_KEEPALIVE_INTERVAL) {
+                auto client_id = pid_to_last_alive.first;
+                pids_to_remove.insert(client_id);
                 LOGGER__INFO("Client disconnected, pid: {}", client_id);
-    syslog(LOG_NOTICE, "Client disconnected, pid: %i", client_id);
+                HAILORT_OS_LOG_INFO("Client disconnected, pid: {}", client_id);
                 ServiceResourceManager<OutputVStream>::get_instance().release_by_pid(client_id);
                 ServiceResourceManager<InputVStream>::get_instance().release_by_pid(client_id);
                 ServiceResourceManager<ConfiguredNetworkGroup>::get_instance().release_by_pid(client_id);
                 ServiceResourceManager<VDevice>::get_instance().release_by_pid(client_id);
+            }
+        }
+        for (auto &pid : pids_to_remove) {
+            m_clients_pids.erase(pid);
+        }
+    }
+}
+
+grpc::Status HailoRtRpcService::client_keep_alive(grpc::ServerContext*, const keepalive_Request *request,
+    empty*)
+{
+    auto client_id = request->pid();
+    std::unique_lock<std::mutex> lock(m_mutex);
+    m_clients_pids[client_id] = std::chrono::high_resolution_clock::now();
     return grpc::Status::OK;
 }
 
@@ -50,6 +81,15 @@ grpc::Status HailoRtRpcService::get_service_version(grpc::ServerContext*, const
     return grpc::Status::OK;
 }
 
+grpc::Status HailoRtRpcService::VDevice_dup_handle(grpc::ServerContext*, const dup_handle_Request *request,
+    dup_handle_Reply* reply)
+{
+    auto &manager = ServiceResourceManager<VDevice>::get_instance();
+    auto handle = manager.dup_handle(request->pid(), request->handle());
+    reply->set_handle(handle);
+    return grpc::Status::OK;
+}
+
 grpc::Status HailoRtRpcService::VDevice_create(grpc::ServerContext *, const VDevice_create_Request *request,
     VDevice_create_Reply *reply)
 {
@@ -64,11 +104,11 @@ grpc::Status HailoRtRpcService::VDevice_create(grpc::ServerContext *, const VDev
     }
 
     hailo_vdevice_params_t params = {
-        .device_count = params_proto.device_count(),
+        params_proto.device_count(),
-        .device_ids = device_ids.data(),
+        device_ids.data(),
-        .scheduling_algorithm = static_cast<hailo_scheduling_algorithm_e>(params_proto.scheduling_algorithm()),
+        static_cast<hailo_scheduling_algorithm_e>(params_proto.scheduling_algorithm()),
-        .group_id = params_proto.group_id().c_str(),
+        params_proto.group_id().c_str(),
-        .multi_process_service = false
+        false
     };
 
     auto vdevice = VDevice::create(params);
@@ -100,7 +140,7 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD
 
     NetworkGroupsParamsMap configure_params_map;
     for (auto &name_configure_params_pair : request->configure_params_map()) {
-        ConfigureNetworkParams network_configure_params;
+        ConfigureNetworkParams network_configure_params{};
         auto proto_configure_params = name_configure_params_pair.params();
         network_configure_params.batch_size = static_cast<uint16_t>(proto_configure_params.batch_size());
         network_configure_params.power_mode = static_cast<hailo_power_mode_t>(proto_configure_params.power_mode());
@@ -110,23 +150,14 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD
         for (auto &proto_name_streams_params_pair : proto_configure_params.stream_params_map()) {
             auto proto_streams_params = proto_name_streams_params_pair.params();
             auto stream_direction = static_cast<hailo_stream_direction_t>(proto_streams_params.direction());
-            hailo_stream_parameters_t stream_params;
+            hailo_stream_parameters_t stream_params{};
+            stream_params.stream_interface = static_cast<hailo_stream_interface_t>(proto_streams_params.stream_interface());
+            stream_params.direction = stream_direction;
+            stream_params.flags = static_cast<hailo_stream_flags_t>(proto_streams_params.flags());
             if (stream_direction == HAILO_H2D_STREAM) {
-                stream_params = {
+                stream_params.pcie_input_params = {0};
-                    .stream_interface = static_cast<hailo_stream_interface_t>(proto_streams_params.stream_interface()),
-                    .direction = stream_direction,
-                    {.pcie_input_params = {
-                        .reserved = 0
-                    }}
-                };
             } else {
-                stream_params = {
+                stream_params.pcie_output_params = {0};
-                    .stream_interface = static_cast<hailo_stream_interface_t>(proto_streams_params.stream_interface()),
-                    .direction = stream_direction,
-                    {.pcie_output_params = {
-                        .reserved = 0
-                    }}
-                };
             }
             network_configure_params.stream_params_by_name.insert({proto_name_streams_params_pair.name(), stream_params});
         }
@@ -135,7 +166,7 @@ grpc::Status HailoRtRpcService::VDevice_configure(grpc::ServerContext*, const VD
         for (auto &proto_name_network_params_pair : proto_configure_params.network_params_map()) {
             auto proto_network_params = proto_name_network_params_pair.params();
             hailo_network_parameters_t net_params {
-                .batch_size = static_cast<uint16_t>(proto_network_params.batch_size())
+                static_cast<uint16_t>(proto_network_params.batch_size())
             };
 
             network_configure_params.network_params_by_name.insert({proto_name_network_params_pair.name(), net_params});
@@ -192,6 +223,15 @@ grpc::Status HailoRtRpcService::VDevice_get_default_streams_interface(grpc::Serv
     return grpc::Status::OK;
 }
 
+grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_dup_handle(grpc::ServerContext*, const dup_handle_Request *request,
+    dup_handle_Reply* reply)
+{
+    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto handle = manager.dup_handle(request->pid(), request->handle());
+    reply->set_handle(handle);
+    return grpc::Status::OK;
+}
+
 grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_release(grpc::ServerContext*, const Release_Request *request,
     Release_Reply *reply)
 {
@@ -410,6 +450,20 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_all_vstream_infos(grp
     return grpc::Status::OK;
 }
 
+grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_is_scheduled(grpc::ServerContext*,
+    const ConfiguredNetworkGroup_is_scheduled_Request *request,
+    ConfiguredNetworkGroup_is_scheduled_Reply *reply)
+{
+    auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng) {
+        return cng->is_scheduled();
+    };
+    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto is_scheduled = manager.execute<bool>(request->handle(), lambda);
+    reply->set_is_scheduled(static_cast<bool>(is_scheduled));
+    reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
+    return grpc::Status::OK;
+}
+
 grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_scheduler_timeout(grpc::ServerContext*,
     const ConfiguredNetworkGroup_set_scheduler_timeout_Request *request,
     ConfiguredNetworkGroup_set_scheduler_timeout_Reply *reply)
@@ -437,6 +491,20 @@ grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_scheduler_threshold(g
     return grpc::Status::OK;
 }
 
+grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_set_scheduler_priority(grpc::ServerContext*,
+    const ConfiguredNetworkGroup_set_scheduler_priority_Request *request,
+    ConfiguredNetworkGroup_set_scheduler_priority_Reply *reply)
+{
+    auto lambda = [](std::shared_ptr<ConfiguredNetworkGroup> cng, uint8_t priority, std::string network_name) {
+            return cng->set_scheduler_priority(priority, network_name);
+    };
+    auto &net_group_manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto status = net_group_manager.execute<hailo_status>(request->handle(), lambda, static_cast<uint8_t>(request->priority()),
+        request->network_name());
+    reply->set_status(status);
+    return grpc::Status::OK;
+}
+
 grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_config_params(grpc::ServerContext*,
     const ConfiguredNetworkGroup_get_config_params_Request *request,
     ConfiguredNetworkGroup_get_config_params_Reply *reply)
@@ -485,11 +553,11 @@ grpc::Status HailoRtRpcService::InputVStreams_create(grpc::ServerContext *, cons
         format.order = hailo_format_order_t(user_buffer_format_proto.order());
         format.type = hailo_format_type_t(user_buffer_format_proto.type());
         hailo_vstream_params_t params = {
-            .user_buffer_format = format,
+            format,
-            .timeout_ms = vstream_params_proto.timeout_ms(),
+            vstream_params_proto.timeout_ms(),
-            .queue_size = vstream_params_proto.queue_size(),
+            vstream_params_proto.queue_size(),
-            .vstream_stats_flags = hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()),
+            hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()),
-            .pipeline_elements_stats_flags = hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags())
+            hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags())
         };
         inputs_params.emplace(param_proto.name(), std::move(params));
     }
@@ -533,11 +601,11 @@ grpc::Status HailoRtRpcService::OutputVStreams_create(grpc::ServerContext *, con
         format.order = hailo_format_order_t(user_buffer_format_proto.order());
         format.type = hailo_format_type_t(user_buffer_format_proto.type());
         hailo_vstream_params_t params = {
-            .user_buffer_format = format,
+            format,
-            .timeout_ms = vstream_params_proto.timeout_ms(),
+            vstream_params_proto.timeout_ms(),
-            .queue_size = vstream_params_proto.queue_size(),
+            vstream_params_proto.queue_size(),
-            .vstream_stats_flags = hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()),
+            hailo_vstream_stats_flags_t(vstream_params_proto.vstream_stats_flags()),
-            .pipeline_elements_stats_flags = hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags())
+            hailo_pipeline_elem_stats_flags_t(vstream_params_proto.pipeline_elements_stats_flags())
         };
         output_params.emplace(param_proto.name(), std::move(params));
     }
@@ -594,11 +662,35 @@ grpc::Status HailoRtRpcService::InputVStream_write(grpc::ServerContext*, const I
     };
     auto &manager = ServiceResourceManager<InputVStream>::get_instance();
     auto status = manager.execute<hailo_status>(request->handle(), lambda, MemoryView::create_const(data.data(), data.size()));
+
+    if (HAILO_STREAM_ABORTED_BY_USER == status) {
+        LOGGER__INFO("User aborted VStream write.");
+        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORTED_BY_USER));
+        return grpc::Status::OK;
+    }
     CHECK_SUCCESS_AS_RPC_STATUS(status,  reply, "VStream write failed");
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));
     return grpc::Status::OK;
 }
 
+grpc::Status HailoRtRpcService::InputVStream_dup_handle(grpc::ServerContext*, const dup_handle_Request *request,
+    dup_handle_Reply *reply)
+{
+    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto handle = manager.dup_handle(request->pid(), request->handle());
+    reply->set_handle(handle);
+    return grpc::Status::OK;
+}
+
+grpc::Status HailoRtRpcService::OutputVStream_dup_handle(grpc::ServerContext*, const dup_handle_Request *request,
+    dup_handle_Reply *reply)
+{
+    auto &manager = ServiceResourceManager<ConfiguredNetworkGroup>::get_instance();
+    auto handle = manager.dup_handle(request->pid(), request->handle());
+    reply->set_handle(handle);
+    return grpc::Status::OK;
+}
+
 grpc::Status HailoRtRpcService::ConfiguredNetworkGroup_get_network_infos(grpc::ServerContext*,
     const ConfiguredNetworkGroup_get_network_infos_Request *request,
     ConfiguredNetworkGroup_get_network_infos_Reply *reply)
@@ -626,6 +718,11 @@ grpc::Status HailoRtRpcService::OutputVStream_read(grpc::ServerContext*, const O
     };
     auto &manager = ServiceResourceManager<OutputVStream>::get_instance();
     auto status = manager.execute<hailo_status>(request->handle(), lambda, MemoryView(data.data(), data.size()));
+    if (HAILO_STREAM_ABORTED_BY_USER == status) {
+        LOGGER__INFO("User aborted VStream read.");
+        reply->set_status(static_cast<uint32_t>(HAILO_STREAM_ABORTED_BY_USER));
+        return grpc::Status::OK;
+    }
     CHECK_SUCCESS_AS_RPC_STATUS(status,  reply, "VStream read failed");
     reply->set_data(data.data(), data.size());
     reply->set_status(static_cast<uint32_t>(HAILO_SUCCESS));

hailort/hailort_service/hailort_rpc_service.hpp

@@ -25,17 +25,21 @@
 #pragma GCC diagnostic pop
 #endif
 
+#include <thread>
+
 namespace hailort
 {
 
 class HailoRtRpcService final : public ProtoHailoRtRpc::Service {
-
 public:
+    HailoRtRpcService();
+
     virtual grpc::Status client_keep_alive(grpc::ServerContext *ctx, const keepalive_Request *request,
         empty*) override;
-
     virtual grpc::Status get_service_version(grpc::ServerContext *, const get_service_version_Request *request,
         get_service_version_Reply *reply) override;
+    virtual grpc::Status VDevice_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request,
+        dup_handle_Reply*) override;
 
     virtual grpc::Status VDevice_create(grpc::ServerContext *, const VDevice_create_Request *request,
         VDevice_create_Reply *reply) override;
@@ -90,6 +94,13 @@ public:
         VStream_get_info_Reply *reply) override;
     virtual grpc::Status OutputVStream_get_info(grpc::ServerContext*, const VStream_get_info_Request *request,
         VStream_get_info_Reply *reply) override;
+    virtual grpc::Status InputVStream_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request,
+        dup_handle_Reply*) override;
+    virtual grpc::Status OutputVStream_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request,
+        dup_handle_Reply*) override;
+
+    virtual grpc::Status ConfiguredNetworkGroup_dup_handle(grpc::ServerContext *ctx, const dup_handle_Request *request,
+        dup_handle_Reply*) override;
     virtual grpc::Status ConfiguredNetworkGroup_release(grpc::ServerContext*, const Release_Request* request,
         Release_Reply* reply) override;
     virtual grpc::Status ConfiguredNetworkGroup_make_input_vstream_params(grpc::ServerContext*,
@@ -122,12 +133,18 @@ public:
     virtual grpc::Status ConfiguredNetworkGroup_get_all_vstream_infos(grpc::ServerContext*,
         const ConfiguredNetworkGroup_get_vstream_infos_Request *request,
         ConfiguredNetworkGroup_get_vstream_infos_Reply *reply) override;
+    virtual grpc::Status ConfiguredNetworkGroup_is_scheduled(grpc::ServerContext*,
+        const ConfiguredNetworkGroup_is_scheduled_Request *request,
+        ConfiguredNetworkGroup_is_scheduled_Reply *reply) override;
     virtual grpc::Status ConfiguredNetworkGroup_set_scheduler_timeout(grpc::ServerContext*,
         const ConfiguredNetworkGroup_set_scheduler_timeout_Request *request,
         ConfiguredNetworkGroup_set_scheduler_timeout_Reply *reply) override;
     virtual grpc::Status ConfiguredNetworkGroup_set_scheduler_threshold(grpc::ServerContext*,
         const ConfiguredNetworkGroup_set_scheduler_threshold_Request *request,
         ConfiguredNetworkGroup_set_scheduler_threshold_Reply *reply) override;
+    virtual grpc::Status ConfiguredNetworkGroup_set_scheduler_priority(grpc::ServerContext*,
+        const ConfiguredNetworkGroup_set_scheduler_priority_Request *request,
+        ConfiguredNetworkGroup_set_scheduler_priority_Reply *reply) override;
     virtual grpc::Status ConfiguredNetworkGroup_get_output_vstream_infos(grpc::ServerContext*,
         const ConfiguredNetworkGroup_get_vstream_infos_Request *request,
         ConfiguredNetworkGroup_get_vstream_infos_Reply *reply) override;
@@ -140,6 +157,13 @@ public:
     virtual grpc::Status ConfiguredNetworkGroup_get_config_params(grpc::ServerContext*,
         const ConfiguredNetworkGroup_get_config_params_Request *request,
         ConfiguredNetworkGroup_get_config_params_Reply *reply) override;
+
+private:
+    void keep_alive();
+
+    std::mutex m_mutex;
+    std::map<uint32_t, std::chrono::time_point<std::chrono::high_resolution_clock>> m_clients_pids;
+    std::unique_ptr<std::thread> m_keep_alive;
 };
 
 }

hailort/hailort_service/hailort_service

@@ -1,4 +1,4 @@
-# This file contains HailoRT's configurable environment variables for HailoRT Service.
+# This file contains HailoRT's configurable environment variables for HailoRT Linux Service.
 # The environment variables are set to their default values.
 # To change an environment variable's value, follow the steps:
 # 1. Change the value of the selected environemt variable in this file
@@ -8,5 +8,4 @@
 [Service]
 HAILORT_LOGGER_PATH="/var/log/hailo"
 HAILO_DISABLE_MULTIPLEXER=0
-HAILO_ENABLE_MULTI_DEVICE_SCHEDULER=0
+HAILO_MONITOR=0
-SCHEDULER_MONITOR=0

hailort/hailort_service/service_resource_manager.hpp

@@ -25,10 +25,9 @@ struct Resource {
     Resource(uint32_t pid, std::shared_ptr<T> resource)
         : pid(pid), resource(std::move(resource))
     {}
-    std::shared_timed_mutex resource_mutex;
+
     uint32_t pid;
     std::shared_ptr<T> resource;
-
 };
 
 template<class T>
@@ -42,37 +41,51 @@ public:
     }
 
     template<class K, class Func, typename... Args>
-    K execute(uint32_t key, Func &lambda, Args... args)
+    K execute(uint32_t handle, Func &lambda, Args... args)
     {
         std::unique_lock<std::mutex> lock(m_mutex);
-        auto resource_expected = resource_lookup(key);
+        auto resource_expected = resource_lookup(handle);
         assert(resource_expected);
-
         auto resource = resource_expected.release();
-        std::shared_lock<std::shared_timed_mutex> resource_lock(resource->resource_mutex);
+
+        assert(contains(m_resources_mutexes, handle));
+        std::shared_lock<std::shared_timed_mutex> resource_lock(m_resources_mutexes[handle]);
         lock.unlock();
         K ret = lambda(resource->resource, args...);
 
         return ret;
     }
 
-    uint32_t register_resource(uint32_t pid, std::shared_ptr<T> const &resource)
+    uint32_t register_resource(uint32_t pid, const std::shared_ptr<T> &resource)
     {
         std::unique_lock<std::mutex> lock(m_mutex);
-
+        auto index = m_current_handle_index.load();
         // Create a new resource and register
-        auto index = m_current_handle_index;
+        m_resources.emplace(m_current_handle_index, std::make_shared<Resource<T>>(pid, std::move(resource)));
-        m_resources.emplace(m_current_handle_index++, std::make_shared<Resource<T>>(pid, std::move(resource)));
+        m_resources_mutexes[m_current_handle_index]; // construct std::shared_timed_mutex
+        m_current_handle_index++;
         return index;
     }
 
-    hailo_status release_resource(uint32_t key)
+    uint32_t dup_handle(uint32_t pid, uint32_t handle)
+    {
+        // Keeping this function for future possible usage
+        (void)pid;
+        return handle;
+    }
+
+    hailo_status release_resource(uint32_t handle)
     {
         std::unique_lock<std::mutex> lock(m_mutex);
-        auto found = m_resources.find(key);
+        auto found = m_resources.find(handle);
-        CHECK(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to release resource with key {}, resource does not exist", key);
+        CHECK(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to release resource with handle {}, resource does not exist", handle);
-        std::unique_lock<std::shared_timed_mutex> resource_lock(found->second->resource_mutex);
+        assert(contains(m_resources_mutexes, handle));
-        m_resources.erase(key);
+        auto resource = m_resources[handle];
+        {
+            std::unique_lock<std::shared_timed_mutex> resource_lock(m_resources_mutexes[handle]);
+            m_resources.erase(handle);
+        }
+        m_resources_mutexes.erase(handle);
         return HAILO_SUCCESS;
     }
 
@@ -80,9 +93,14 @@ public:
     {
         std::unique_lock<std::mutex> lock(m_mutex);
         for (auto iter = m_resources.begin(); iter != m_resources.end(); ) {
+            auto handle = iter->first;
             if (iter->second->pid == pid) {
-                std::unique_lock<std::shared_timed_mutex> resource_lock(iter->second->resource_mutex);
+                assert(contains(m_resources_mutexes, handle));
+                {
+                    std::unique_lock<std::shared_timed_mutex> resource_lock(m_resources_mutexes[handle]);
                     iter = m_resources.erase(iter);
+                }
+                m_resources_mutexes.erase(handle);
             } else {
                 ++iter;
             }
@@ -94,18 +112,18 @@ private:
         : m_current_handle_index(0)
     {}
 
-    Expected<std::shared_ptr<Resource<T>>> resource_lookup(uint32_t key)
+    Expected<std::shared_ptr<Resource<T>>> resource_lookup(uint32_t handle)
     {
-        auto found = m_resources.find(key);
+        auto found = m_resources.find(handle);
-        CHECK_AS_EXPECTED(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to find resource with key {}", key);
+        CHECK_AS_EXPECTED(found != m_resources.end(), HAILO_NOT_FOUND, "Failed to find resource with handle {}", handle);
-
         auto resource = found->second;
         return resource;
     }
 
     std::mutex m_mutex;
-    uint32_t m_current_handle_index;
+    std::atomic<uint32_t> m_current_handle_index;
     std::unordered_map<uint32_t, std::shared_ptr<Resource<T>>> m_resources;
+    std::unordered_map<uint32_t, std::shared_timed_mutex> m_resources_mutexes;
 };
 
 }

hailort/hailort_service/unix/hailort_service.cpp

@@ -4,9 +4,7 @@
  *
  * @file hailort_service.cpp
  * @brief main for hailort service 
- * To run as without daemonize the executable:
+ * To run without daemonization run the hailort_service executable with `standalone`.
- *       1) Compile with `./build.sh`
- *       2) Run `./bin/linux.x86_64.debug/hailort_service standalone`
  *
  * To run as daemon service please follow the steps:
  *       1) Install the HailoRT:
@@ -27,12 +25,13 @@
 #include "common/utils.hpp"
 #include "common/filesystem.hpp"
 #include "hailo/hailort_common.hpp"
+#include "common/os_utils.hpp"
 
 #include <syslog.h>
 #include <sys/stat.h>
 
 void RunService() {
-    std::string server_address(hailort::HAILO_DEFAULT_UDS_ADDR);
+    const std::string server_address = hailort::HAILORT_SERVICE_DEFAULT_ADDR;
     hailort::HailoRtRpcService service;
 
     grpc::ServerBuilder builder;
@@ -48,20 +47,20 @@ void write_pid_to_lock_file()
 {
     auto status = hailort::Filesystem::create_directory(HAILO_DAEMON_PID_DIR);
     if (status != HAILO_SUCCESS) {
-        syslog(LOG_ERR, "Cannot create directory at path, status=%i", status);
+        HAILORT_OS_LOG_ERROR("Cannot create directory at path, status={}", status);
         return;
     }
 
     auto locked_file = hailort::LockedFile::create(HAILO_DAEMON_PID_FILE, "wx");
     if (HAILO_SUCCESS != locked_file.status()) {
-        syslog(LOG_ERR, "Failed to lock pid file for hailort service, status=%i", locked_file.status());
+        HAILORT_OS_LOG_ERROR("Failed to lock pid file for hailort service, status={}", locked_file.status());
         return;
     }
 
     std::string pid = std::to_string(getpid());
     auto ret = write(locked_file->get_fd(), pid.c_str(), pid.size());
     if (-1 == ret) {
-        syslog(LOG_ERR, "Failed to write pid to lock file for hailort service, errno=%i", errno);
+        HAILORT_OS_LOG_ERROR("Failed to write pid to lock file for hailort service, errno={}", errno);
         return;
     }
 }
@@ -72,7 +71,7 @@ int main(int argc, char *argv[])
     if (!is_standalone) {
         int ret = daemon(0, 0);
         if (ret < 0) {
-            syslog(LOG_ERR, "Failed to create daemon with errno %i", errno);
+            HAILORT_OS_LOG_ERROR("Failed to create daemon with errno {}", errno);
             exit(EXIT_FAILURE);
         }
 

hailort/hailort_service/windows/hailort_service.cpp

@@ -0,0 +1,246 @@
+/**
+ * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ *
+ * @file hailort_service.cpp
+ * @brief main for hailort service 
+ * The service code is based on Microsoft's documenataion: https://learn.microsoft.com/en-us/windows/win32/services/the-complete-service-sample
+ *
+ * Running hailort_service:
+ * To run hailort_service without Windows service control manager (SCM), run hailort_service executable with `standalone`.
+ *
+ * To run as a service application please follow the steps:
+ *       1) Compile and install libhailort:
+ *          `cmake -H. -Bbuild -A=x64 -DCMAKE_BUILD_TYPE=Release -DHAILO_BUILD_SERVICE=1 && cmake --build build --config release --target install`
+ * 
+ *       2) To install the service, run the `hailort_service` executable with `install`:
+ *          `"C:\Program Files\HailoRT\bin\hailort_service.exe" install`
+
+ *       3) Start the service:
+ *           `sc start hailort_service`
+ *
+ *       4) Stop the service:
+ *           `sc stop hailort_service`
+ *
+ *       5) Delete service:
+ *           `sc delete hailort_service`
+*/
+
+#include "hailort_rpc_service.hpp"
+#include "rpc/rpc_definitions.hpp"
+#include "common/os_utils.hpp"
+
+#include <winsvc.h>
+#include <windows.h>
+#include <tchar.h>
+#include <strsafe.h>
+
+#define SERVICE_NAME ("hailort_service")
+static const DWORD HRT_SERVICE_INIT_WAIT_TIME_MS(3000);
+static const DWORD HRT_SERVICE_ZERO_WAIT_TIME_MS(0);
+
+SERVICE_STATUS g_service_status = {0};
+SERVICE_STATUS_HANDLE g_service_status_handle = nullptr;
+HANDLE g_stop_event_handle = INVALID_HANDLE_VALUE;
+std::unique_ptr<grpc::Server> g_hailort_rpc_server = nullptr;
+
+void RunService()
+{
+    const std::string server_address = hailort::HAILORT_SERVICE_DEFAULT_ADDR;
+    hailort::HailoRtRpcService service;
+
+    grpc::ServerBuilder builder;
+    builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
+    builder.SetMaxReceiveMessageSize(-1);
+    builder.RegisterService(&service);
+    g_hailort_rpc_server = builder.BuildAndStart();
+    g_hailort_rpc_server->Wait();
+}
+
+// Installs the service in the SCM database
+void install_service()
+{
+    SC_HANDLE open_sc_manager_handle = nullptr;
+    SC_HANDLE create_service_handle = nullptr;
+    TCHAR module_path[MAX_PATH];
+
+    if (!GetModuleFileName(nullptr, module_path, MAX_PATH)) {
+        HAILORT_OS_LOG_ERROR("GetModuleFileName() failed. Cannot install hailort service, LE = {}", GetLastError());
+        return;
+    }
+
+    TCHAR quoted_module_path[MAX_PATH];
+    StringCbPrintf(quoted_module_path, MAX_PATH, ("\"%s\""), module_path);
+
+    // Get a handle to the SCM database.
+    open_sc_manager_handle = OpenSCManager(
+        nullptr,                    // local computer
+        nullptr,                    // ServicesActive database 
+        SC_MANAGER_ALL_ACCESS);     // full access rights 
+
+    if (nullptr == open_sc_manager_handle) {
+        HAILORT_OS_LOG_ERROR("OpenSCManager() failed. Cannot install hailort service, LE = {}", GetLastError());
+        return;
+    }
+
+    // Create the service
+    create_service_handle = CreateService(
+        open_sc_manager_handle,         // SCM database
+        SERVICE_NAME,                   // name of service
+        SERVICE_NAME,                   // service name to display
+        SERVICE_ALL_ACCESS,             // desired access
+        SERVICE_WIN32_OWN_PROCESS,      // service type
+        SERVICE_DEMAND_START,           // start type
+        SERVICE_ERROR_NORMAL,           // error control type
+        quoted_module_path,             // path to service's binary
+        nullptr,                        // no load ordering group
+        nullptr,                        // no tag identifier
+        nullptr,                        // no dependencies
+        nullptr,                        // LocalSystem account
+        nullptr);                       // no password
+ 
+    if (nullptr == create_service_handle) {
+        HAILORT_OS_LOG_ERROR("CreateService() failed. Cannot install hailort service, LE = {}", GetLastError());
+        CloseServiceHandle(open_sc_manager_handle);
+        return;
+    }
+
+    CloseServiceHandle(create_service_handle); 
+    CloseServiceHandle(open_sc_manager_handle);
+}
+
+// Sets the current service status and reports it to the SCM
+void report_service_status(DWORD current_state, DWORD win32_exit_code, DWORD wait_hint)
+{
+    static DWORD check_point = 1;
+    g_service_status.dwCurrentState = current_state;
+    g_service_status.dwWin32ExitCode = win32_exit_code;
+    g_service_status.dwWaitHint = wait_hint;
+
+    if (SERVICE_START_PENDING == current_state) {
+        // Service is about to start
+        g_service_status.dwControlsAccepted = 0;
+    } else {
+        g_service_status.dwControlsAccepted = SERVICE_ACCEPT_STOP;  
+    }
+
+    if ((SERVICE_RUNNING == current_state) || (SERVICE_STOPPED == current_state)) {
+        g_service_status.dwCheckPoint = 0;
+    } else {
+        g_service_status.dwCheckPoint = check_point++;
+    }
+
+    // Report the service status to the SCM.
+    SetServiceStatus(g_service_status_handle, &g_service_status);
+}
+
+// Called by SCM whenever a control code is sent to the service
+void control_handler(DWORD control_code)
+{
+    switch(control_code) {
+        case SERVICE_CONTROL_STOP:
+            report_service_status(SERVICE_STOP_PENDING, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS);
+
+            // Signal the service to stop.
+            SetEvent(g_stop_event_handle);
+            report_service_status(SERVICE_STOPPED, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS);
+            return;
+
+        default: 
+            break;
+    }
+}
+
+void terminate_server_thread(HANDLE thread_handle)
+{
+    g_hailort_rpc_server->Shutdown();
+    auto rpc_server_wait_res = WaitForSingleObject(thread_handle, INFINITE);
+    if (WAIT_OBJECT_0 == rpc_server_wait_res) {
+        CloseHandle(thread_handle);
+    } else {
+        HAILORT_OS_LOG_ERROR("Failed waiting on hailort server thread, LE = {}", GetLastError());
+        report_service_status(SERVICE_STOPPED, GetLastError(), HRT_SERVICE_ZERO_WAIT_TIME_MS);
+    }
+}
+
+// The service code
+void service_init()
+{
+    // Create an event. The control handler function signals this event when it receives the stop control code.
+    g_stop_event_handle = CreateEvent(
+        nullptr,       // default security attributes
+        TRUE,          // manual reset event
+        FALSE,         // not signaled
+        nullptr);      // no name
+    if (nullptr == g_stop_event_handle) {
+        report_service_status(SERVICE_STOPPED, GetLastError(), HRT_SERVICE_ZERO_WAIT_TIME_MS);
+        return;
+    }
+
+    // Report SCM the running status when initialization is complete.
+    report_service_status(SERVICE_RUNNING, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS);
+
+    // Start a thread that will perform the main task of the service
+    HANDLE service_thread_handle = CreateThread(nullptr, 0, (LPTHREAD_START_ROUTINE)RunService, nullptr, 0, nullptr);
+    if (nullptr == service_thread_handle) {
+        HAILORT_OS_LOG_ERROR("Failed to create hailort_service thread, LE = {}", GetLastError());
+    }
+
+    // Wait for stop service signal
+    auto service_wait_res = WaitForSingleObject(g_stop_event_handle, INFINITE);
+    if (WAIT_OBJECT_0 == service_wait_res) {
+        terminate_server_thread(service_thread_handle);
+        report_service_status(SERVICE_STOPPED, NO_ERROR, HRT_SERVICE_ZERO_WAIT_TIME_MS);
+    } else {
+        HAILORT_OS_LOG_ERROR("Failed waiting for signal on hailort_service stop event, LE = {}", GetLastError());
+        report_service_status(SERVICE_STOPPED, GetLastError(), HRT_SERVICE_ZERO_WAIT_TIME_MS);
+    }
+    CloseHandle(g_stop_event_handle);
+}
+
+// Entry point for the service
+void service_main(DWORD /* dwArgc */, LPTSTR * /*lpszArgv*/)
+{
+    // Register the handler function for the service
+    g_service_status_handle = RegisterServiceCtrlHandler(SERVICE_NAME, control_handler);
+    if (!g_service_status_handle) {
+        HAILORT_OS_LOG_ERROR("RegisterServiceCtrlHandler() failed. Cannot start hailort service, LE = {}", GetLastError()); 
+        return;
+    }
+
+    g_service_status.dwServiceType = SERVICE_WIN32_OWN_PROCESS; 
+    g_service_status.dwServiceSpecificExitCode = 0;
+
+    // Report initial status to the SCM - service is starting
+    report_service_status(SERVICE_START_PENDING, NO_ERROR, HRT_SERVICE_INIT_WAIT_TIME_MS);
+    service_init();
+}
+
+int main(int argc, TCHAR *argv[])
+{ 
+    const bool is_standalone = ((1 < argc) && (0 == lstrcmpi(argv[1], "standalone")));
+    if (is_standalone) {
+        RunService();
+        return 0;
+    }
+
+    // If command-line parameter is "install", install the service. 
+    // Otherwise, the service is probably being started by the SCM.
+    if ((0 < argc) && (0 == lstrcmpi(argv[1], "install"))) {
+        install_service();
+        return 0;
+    }
+
+    // Service is being started by the SCM
+    SERVICE_TABLE_ENTRY dispatch_table[] = { 
+        {SERVICE_NAME, static_cast<LPSERVICE_MAIN_FUNCTION>(service_main)},
+        {nullptr, nullptr}
+    };
+ 
+    // This call returns when the service has stopped (SERVICE_STOPPED).
+    // The process should simply terminate when the call returns.
+    if (!StartServiceCtrlDispatcher(dispatch_table)) {
+        HAILORT_OS_LOG_ERROR("StartServiceCtrlDispatcher() failed. Cannot start hailort service, LE = {}", GetLastError());
+    }
+    return 0;
+}

hailort/hailort_service/windows/hailort_service_env_vars.bat

@@ -0,0 +1,11 @@
+@REM This file contains HailoRT's configurable environment variables for HailoRT Windows Service.
+@REM  The environment variables are set to their default values, and are seperated by the character \0.
+@REM  To change an environment variable's value, follow the steps:
+@REM  1. Change the value of the selected environment variable in this file
+@REM  2. Run this script
+@REM  3. Restart the service
+@REM Running this script requires Administrator permissions.
+
+reg ADD HKLM\SYSTEM\CurrentControlSet\Services\hailort_service /f /v Environment /t REG_MULTI_SZ /d ^
+HAILORT_LOGGER_PATH="%PROGRAMDATA%\HailoRT_Service\logs"\0^
+HAILO_DISABLE_MULTIPLEXER=0\0

hailort/hailortcli/CMakeLists.txt

@@ -20,7 +20,6 @@ set(HAILORTCLI_CPP_FILES
     fw_config_serializer.cpp
     common.cpp
     benchmark_command.cpp
-    temp_measurement.cpp
     parse_hef_command.cpp
     graph_printer.cpp
     mon_command.cpp
@@ -30,6 +29,7 @@ set(HAILORTCLI_CPP_FILES
     run2/live_printer.cpp
     run2/timer_live_track.cpp
     run2/network_live_track.cpp
+    run2/measurement_live_track.cpp
     )
     
 if(UNIX)
@@ -53,9 +53,10 @@ add_executable(hailortcli
     ${HAILORT_COMMON_CPP_SOURCES}
     ${PROJECT_SOURCE_DIR}/common/src/firmware_header_utils.c
     ${PROJECT_SOURCE_DIR}/common/src/md5.c
-    ${HAILORT_SRC_DIR}/pipeline.cpp
+    ${HAILORT_SRC_DIR}/net_flow/pipeline/pipeline.cpp # TODO: link dynamically with libhailort
-    ${HAILO_FULL_OS_DIR}/event.cpp
+    ${HAILO_FULL_OS_DIR}/event.cpp # TODO: link dynamically with libhailort
 )
+
 target_compile_options(hailortcli PRIVATE ${HAILORT_COMPILE_OPTIONS})
 set_property(TARGET hailortcli PROPERTY CXX_STANDARD 14)
 set_property(TARGET hailortcli PROPERTY INSTALL_RPATH "$ORIGIN" "../lib/") # Link with a relative libhailort

hailort/hailortcli/common.cpp

@@ -69,15 +69,14 @@ Expected<std::string> CliCommon::current_time_to_string()
 void CliCommon::reset_cursor(size_t lines_count)
 {
     for (size_t i = 0; i < lines_count; i++) {
-        std::cout << FORMAT_CURSOR_UP_LINE;
+        std::cout << FORMAT_CURSOR_UP_LINE; // Override prev line
+        std::cout << FORMAT_CLEAR_LINE; // Delete line
     }
 }
 
-void CliCommon::clear_lines_down(size_t lines_count)
+void CliCommon::clear_terminal()
 {
-    for (size_t i = 0; i < lines_count; i++) {
+    std::cout << FORMAT_CLEAR_TERMINAL_CURSOR_FIRST_LINE;
-        std::cout << FORMAT_CURSOR_DOWN_CLEAR_LINE;
-    }
 }
 
 bool CliCommon::is_positive_number(const std::string &s)
@@ -91,3 +90,14 @@ bool CliCommon::is_non_negative_number(const std::string &s)
     bool is_number = (!s.empty()) && (std::all_of(s.begin(), s.end(), ::isdigit));
     return is_number && (0 <= std::stoi(s));
 }
+
+AlternativeTerminal::AlternativeTerminal()
+{
+    std::cout << FORMAT_ENTER_ALTERNATIVE_SCREEN;
+    CliCommon::clear_terminal();
+}
+
+AlternativeTerminal::~AlternativeTerminal()
+{
+    std::cout << FORMAT_EXIT_ALTERNATIVE_SCREEN;
+}

hailort/hailortcli/common.hpp

@@ -22,7 +22,11 @@ using namespace hailort;
 // http://www.climagic.org/mirrors/VT100_Escape_Codes.html
 #define FORMAT_CLEAR_LINE "\033[2K\r"
 #define FORMAT_CURSOR_UP_LINE "\033[F"
-#define FORMAT_CURSOR_DOWN_CLEAR_LINE "\033[B\33[2K\r"
+#define FORMAT_CLEAR_TERMINAL_CURSOR_FIRST_LINE "\033[2J\033[1;1H"
+#define FORMAT_ENTER_ALTERNATIVE_SCREEN "\033[?1049h"
+#define FORMAT_EXIT_ALTERNATIVE_SCREEN "\033[?1049l"
+#define FORMAT_GREEN_PRINT "\x1B[1;32m"
+#define FORMAT_NORMAL_PRINT "\x1B[0m"
 
 class CliCommon final
 {
@@ -32,9 +36,9 @@ public:
     static std::string duration_to_string(std::chrono::seconds secs);
     static Expected<std::string> current_time_to_string();
     static void reset_cursor(size_t number_of_lines);
-    static void clear_lines_down(size_t number_of_lines);
     static bool is_positive_number(const std::string &s);
     static bool is_non_negative_number(const std::string &s);
+    static void clear_terminal();
 };
 
 // Validators
@@ -54,6 +58,14 @@ struct FileSuffixValidator : public CLI::Validator {
     }
 };
 
+// This class is an RAII for running in alternative terminal
+class AlternativeTerminal final
+{
+public:
+    AlternativeTerminal();
+    ~AlternativeTerminal();
+};
+
 // Based on NLOHMANN_JSON_SERIALIZE_ENUM (json/include/nlohmann/json.hpp)
 // Accepts a static array instead of building one in the function
 #define NLOHMANN_JSON_SERIALIZE_ENUM2(ENUM_TYPE, _pair_arr)\

hailort/hailortcli/download_action_list_command.cpp

@@ -20,7 +20,7 @@
 // TODO - HRT-7364 - add CPU subsystem frequency into the device extended info control
 // and use it for get the timer's frequency
 #define NN_CORE_TO_TIMER_FREQ_FACTOR (2)
-#define MERCURY_VPU_CORE_CPU_DEFAULT_FREQ_MHZ (200)
+#define HAILO15_VPU_CORE_CPU_DEFAULT_FREQ_MHZ (200)
 
 constexpr int DownloadActionListCommand::INVALID_NUMERIC_VALUE;
 
@@ -44,9 +44,9 @@ hailo_status DownloadActionListCommand::execute(Device &device, const std::strin
     auto chip_arch = device.get_architecture();
     CHECK_EXPECTED_AS_STATUS(chip_arch);
     unsigned int clock_cycle = 0;
-    // TODO - HRT-8046 Implement extended device info for mercury
+    // TODO - HRT-8046 Implement extended device info for hailo15
-    if (HAILO_ARCH_MERCURY_VPU == chip_arch.value()) {
+    if (HAILO_ARCH_HAILO15 == chip_arch.value()) {
-        clock_cycle = MERCURY_VPU_CORE_CPU_DEFAULT_FREQ_MHZ;
+        clock_cycle = HAILO15_VPU_CORE_CPU_DEFAULT_FREQ_MHZ;
     } else {
         auto extended_info = device.get_extended_device_information();
         CHECK_EXPECTED_AS_STATUS(extended_info);

hailort/hailortcli/fw_control_command.cpp

@@ -138,10 +138,8 @@ static std::string identity_arch_string(const hailo_device_identity_t &identity)
         return "HAILO8";
     case HAILO_ARCH_HAILO8L:
         return "HAILO8L";
-    case HAILO_ARCH_MERCURY_CA:
+    case HAILO_ARCH_HAILO15:
-        return "MERCURY_CA";
+        return "HAILO15";
-    case HAILO_ARCH_MERCURY_VPU:
-        return "MERCURY_VPU";
     default:
         return "Unknown";
     }

hailort/hailortcli/fw_logger_command.cpp

@@ -67,7 +67,7 @@ hailo_status FwLoggerCommand::execute_on_device(Device &device)
         return HAILO_INVALID_OPERATION;
     }
     
-    if (Device::Type::CORE != device.get_type()) {
+    if (Device::Type::INTEGRATED != device.get_type()) {
         status = write_logs_to_file(device, ofs, HAILO_CPU_ID_0);
         if (status != HAILO_SUCCESS){
             return status;

hailort/hailortcli/graph_printer.hpp

@@ -12,7 +12,8 @@
 
 #include "hailo/hailort.h"
 #include "hailo/vstream.hpp"
-#include "pipeline.hpp"
+
+#include "net_flow/pipeline/pipeline.hpp"
 
 #include "DotWriter.h"
 
@@ -20,6 +21,7 @@
 #include <memory>
 #include <type_traits>
 
+
 namespace hailort
 {
 

hailort/hailortcli/hailortcli.hpp

@@ -27,6 +27,9 @@ using namespace hailort;
         }                                                                       \
     } while (0)
 
+// Used for run and run2 commands
+constexpr size_t OVERALL_LATENCY_TIMESTAMPS_LIST_LENGTH (512);
+
 struct hailo_device_params {
     std::vector<std::string> device_ids;
 };

hailort/hailortcli/infer_stats_printer.cpp

@@ -10,7 +10,8 @@
 #include "infer_stats_printer.hpp"
 #include "run_command.hpp"
 #include "common.hpp"
-#include "pipeline.hpp"
+
+#include "net_flow/pipeline/pipeline.hpp"
 
 #include <fstream>
 #include <iostream>
@@ -220,11 +221,17 @@ void InferStatsPrinter::print_csv(const std::vector<std::string> &network_groups
                 for (const auto &pair : inference_result->m_temp_measurements) {
                     if (nullptr != pair.second) {
                         m_results_csv_file << ",";
-                        m_results_csv_file << pair.second->min_value;
+                        if (auto min = pair.second->min()) {
+                            m_results_csv_file << *min;
+                        }
                         m_results_csv_file << ",";
-                        m_results_csv_file << pair.second->average_value;
+                        if (auto mean = pair.second->mean()) {
+                            m_results_csv_file << *mean;
+                        }
                         m_results_csv_file << ",";
-                        m_results_csv_file << pair.second->max_value;
+                        if (auto max = pair.second->max()) {
+                            m_results_csv_file << *max;
+                        }
                     } else {
                         m_results_csv_file << ",,,";
                     }
@@ -447,9 +454,15 @@ void InferStatsPrinter::print_stdout(Expected<InferResult> &inference_result)
         }
         auto temp_measure_iter = inference_result->m_temp_measurements.find(pair.first);
         if ((temp_measure_iter != inference_result->m_temp_measurements.end()) && (nullptr != temp_measure_iter->second)) {
-            measurement_stream << "    Minimum chip temperature: " << temp_measure_iter->second->min_value << "C" << std::endl;
+            if (auto min = temp_measure_iter->second->min()) {
-            measurement_stream << "    Average chip temperature: " << temp_measure_iter->second->average_value << "C" << std::endl;
+                measurement_stream << "    Minimum chip temperature: " << *min << "C" << std::endl;
-            measurement_stream << "    Maximum chip temperature: " << temp_measure_iter->second->max_value << "C" << std::endl;
+            }
+            if (auto mean = temp_measure_iter->second->mean()) {
+                measurement_stream << "    Average chip temperature: " << *mean << "C" << std::endl;
+            }
+            if (auto max = temp_measure_iter->second->max()) {
+                measurement_stream << "    Maximum chip temperature: " << *max << "C" << std::endl;
+            }
         }
         if (0 != measurement_stream.rdbuf()->in_avail()) {
             std::cout << "  Device: " << pair.first << std::endl;

hailort/hailortcli/inference_progress.cpp

@@ -9,6 +9,7 @@
 #include "inference_progress.hpp"
 #include "infer_stats_printer.hpp"
 #include "common.hpp"
+#include "common/os_utils.hpp"
 
 #include <iostream>
 #include <iomanip>
@@ -40,6 +41,7 @@ InferProgress::InferProgress(const inference_runner_params &params,
 void InferProgress::start()
 {
     m_print_thread = std::thread([this] () {
+        OsUtils::set_current_thread_name("PROGRESS_BAR");
         while (true) {
             print_progress(true);
             auto status = m_stop_event->wait(m_print_interval);

hailort/hailortcli/inference_result.hpp

@@ -11,7 +11,7 @@
 #define _HAILO_INFER_RESULT_
 
 #include "power_measurement_command.hpp"
-#include "temp_measurement.hpp"
+#include "common/device_measurements.hpp"
 #include "hailo/runtime_statistics.hpp"
 #include "hailo/vstream.hpp"
 
@@ -309,7 +309,7 @@ public:
         for (const auto &device : devices) {
             m_power_measurements.emplace(device.get().get_dev_id(), std::shared_ptr<LongPowerMeasurement>{});
             m_current_measurements.emplace(device.get().get_dev_id(), std::shared_ptr<LongPowerMeasurement>{});
-            m_temp_measurements.emplace(device.get().get_dev_id(), std::shared_ptr<TempMeasurementData>{});
+            m_temp_measurements.emplace(device.get().get_dev_id(), std::shared_ptr<AccumulatorResults>{});
         }
     }
 
@@ -329,7 +329,7 @@ public:
         return HAILO_SUCCESS;
     }
 
-    hailo_status set_temp_measurement(const std::string &device_id, std::shared_ptr<TempMeasurementData> &&temp_measure)
+    hailo_status set_temp_measurement(const std::string &device_id, std::shared_ptr<AccumulatorResults> &&temp_measure)
     {
         auto iter = m_temp_measurements.find(device_id);
         CHECK(m_temp_measurements.end() != iter, HAILO_INVALID_ARGUMENT);
@@ -341,7 +341,7 @@ public:
     // TODO: create a struct containing all device measurements, and keep only one map
     std::map<std::string, std::shared_ptr<LongPowerMeasurement>> m_power_measurements;
     std::map<std::string, std::shared_ptr<LongPowerMeasurement>> m_current_measurements;
-    std::map<std::string, std::shared_ptr<TempMeasurementData>> m_temp_measurements;
+    std::map<std::string, std::shared_ptr<AccumulatorResults>> m_temp_measurements;
 
 private:
     std::vector<NetworkGroupInferResult> m_network_group_results;

hailort/hailortcli/mon_command.cpp

@@ -7,12 +7,16 @@
  * @brief Monitor of networks - Presents information about the running networks
  **/
 
-#include "mon_command.hpp"
-#include "common.hpp"
 #include "hailo/hailort.h"
+
 #include "common/filesystem.hpp"
 
+#include "mon_command.hpp"
+#include "common.hpp"
+
 #include <iostream>
+#include <signal.h>
+#include <thread>
 #if defined(__GNUC__)
 #include <sys/ioctl.h>
 #endif
@@ -20,15 +24,22 @@
 namespace hailort
 {
 
-// TODO: Deal with longer networks names - should use HAILO_MAX_NETWORK_NAME_SIZE but its too long for one line
+constexpr size_t STRING_WIDTH = 60;
-constexpr size_t NETWORK_NAME_WIDTH = 40;
+constexpr size_t NETWORK_GROUP_NAME_WIDTH = STRING_WIDTH;
-constexpr size_t STREAM_NAME_WIDTH = 60;
+constexpr size_t DEVICE_ID_WIDTH = STRING_WIDTH;
-constexpr size_t ACTIVE_TIME_WIDTH = 25;
+constexpr size_t STREAM_NAME_WIDTH = STRING_WIDTH;
+constexpr size_t UTILIZATION_WIDTH = 25;
 constexpr size_t NUMBER_WIDTH = 15;
 constexpr size_t TERMINAL_DEFAULT_WIDTH = 80;
-constexpr size_t LINE_LENGTH = 125;
+constexpr size_t LINE_LENGTH = NETWORK_GROUP_NAME_WIDTH + STREAM_NAME_WIDTH + UTILIZATION_WIDTH + NUMBER_WIDTH;
 constexpr std::chrono::milliseconds EPSILON_TIME(500);
 
+inline std::string truncate_str(const std::string &original_str, uint32_t max_length)
+{
+    static const std::string ELLIPSIS = "...  ";
+    return (original_str.length() > max_length) ? original_str.substr(0, (max_length - ELLIPSIS.length())) + ELLIPSIS : original_str;
+}
+
 MonCommand::MonCommand(CLI::App &parent_app) :
     Command(parent_app.add_subcommand("monitor", "Monitor of networks - Presents information about the running networks. " \
      "To enable monitor, set in the application process the environment variable '" + std::string(SCHEDULER_MON_ENV_VAR) + "' to 1."))
@@ -40,65 +51,86 @@ hailo_status MonCommand::execute()
     LOGGER__ERROR("hailortcli `monitor` command is not supported on Windows");
     return HAILO_NOT_IMPLEMENTED;
 #else
-    return print_table();
+    return run_monitor();
 #endif
 }
 
-size_t MonCommand::print_networks_info_header()
+void MonCommand::print_devices_info_header()
 {
     std::cout << 
-        std::setw(NETWORK_NAME_WIDTH) << std::left << "Network" <<
+        std::setw(DEVICE_ID_WIDTH) << std::left << "Device ID" <<
-        std::setw(NUMBER_WIDTH) << std::left << "FPS" <<
+        std::setw(UTILIZATION_WIDTH) << std::left << "Utilization (%)" <<
-        std::setw(ACTIVE_TIME_WIDTH) << std::left << "Active Time (%) " <<
+        std::setw(STRING_WIDTH) << std::left << "Architecture" <<
-        std::setw(NUMBER_WIDTH) << std::left << "PID" << 
         "\n" << std::left << std::string(LINE_LENGTH, '-') << "\n";
-    static const uint32_t header_lines_count = 2;
-
-    return header_lines_count;
 }
 
-size_t MonCommand::print_networks_info_table(const ProtoMon &mon_message)
+void MonCommand::print_devices_info_table(const ProtoMon &mon_message)
+{
+    auto data_line_len = NUMBER_WIDTH + NETWORK_GROUP_NAME_WIDTH + DEVICE_ID_WIDTH;
+    auto rest_line_len = LINE_LENGTH - data_line_len;
+
+    for (const auto &device_info : mon_message.device_infos()) {
+        auto device_id = device_info.device_id();
+        auto utilization = device_info.utilization();
+        auto device_arch = device_info.device_arch();
+
+        std::cout << std::setprecision(1) << std::fixed <<
+            std::setw(DEVICE_ID_WIDTH) << std::left << device_id <<
+            std::setw(UTILIZATION_WIDTH) << std::left << utilization <<
+            std::setw(STRING_WIDTH) << std::left << device_arch <<
+            std::string(rest_line_len, ' ') << "\n";
+    }
+}
+
+void MonCommand::print_networks_info_header()
+{
+    std::cout << 
+        std::setw(NETWORK_GROUP_NAME_WIDTH) << std::left << "Model" <<
+        std::setw(UTILIZATION_WIDTH) << std::left << "Utilization (%) " <<
+        std::setw(NUMBER_WIDTH) << std::left << "FPS" <<
+        std::setw(NUMBER_WIDTH) << std::left << "PID" << 
+        "\n" << std::left << std::string(LINE_LENGTH, '-') << "\n";
+}
+
+void MonCommand::print_networks_info_table(const ProtoMon &mon_message)
 {
     const uint32_t NUMBER_OBJECTS_COUNT = 3;
-    auto data_line_len = (NUMBER_WIDTH * NUMBER_OBJECTS_COUNT) + NETWORK_NAME_WIDTH;
+    auto data_line_len = (NUMBER_WIDTH * NUMBER_OBJECTS_COUNT) + NETWORK_GROUP_NAME_WIDTH;
     auto rest_line_len = LINE_LENGTH - data_line_len;
 
     const std::string &pid = mon_message.pid();
-    for (auto net_info : mon_message.networks_infos()) {
+    for (const auto &net_info : mon_message.networks_infos()) {
-        auto &net_name = net_info.network_name();
+        auto &original_net_name = net_info.network_name();
+        auto net_name = truncate_str(original_net_name, NETWORK_GROUP_NAME_WIDTH);
         auto fps = net_info.fps();
-        auto active_time = net_info.active_time();
+        auto utilization = net_info.utilization();
 
         std::cout << std::setprecision(1) << std::fixed <<
-            std::setw(NETWORK_NAME_WIDTH) << std::left << net_name <<
+            std::setw(STRING_WIDTH) << std::left << net_name <<
+            std::setw(UTILIZATION_WIDTH) << std::left << utilization <<
             std::setw(NUMBER_WIDTH) << std::left << fps <<
-            std::setw(ACTIVE_TIME_WIDTH) << std::left << active_time <<
             std::setw(NUMBER_WIDTH) << std::left << pid << std::string(rest_line_len, ' ') << "\n";
     }
-
-    return mon_message.networks_infos().size();
 }
 
-size_t MonCommand::print_frames_header()
+void MonCommand::print_frames_header()
 {
     std::cout << 
-        std::setw(NETWORK_NAME_WIDTH) << std::left << "Network" <<
+        std::setw(STRING_WIDTH) << std::left << "Model" <<
-        std::setw(STREAM_NAME_WIDTH) << std::left << "Stream" <<
+        std::setw(STRING_WIDTH) << std::left << "Stream" <<
         std::setw(NUMBER_WIDTH) << std::left << "Direction" << 
         std::setw(NUMBER_WIDTH) << std::left << "Frames" << 
         "\n" << std::left << std::string(LINE_LENGTH, '-') << "\n";
-    static const size_t header_lines_count = 2;
-    return header_lines_count;
 }
 
-size_t MonCommand::print_frames_table(const ProtoMon &mon_message)
+void MonCommand::print_frames_table(const ProtoMon &mon_message)
 {
-    size_t table_lines_count = 0;
+    for (const auto &net_info : mon_message.net_frames_infos()) {
-    for (auto &net_info : mon_message.net_frames_infos()) {
+        auto &original_net_name = net_info.network_name();
-        auto &net_name = net_info.network_name();
+        auto net_name = truncate_str(original_net_name, NETWORK_GROUP_NAME_WIDTH);
-        table_lines_count += net_info.streams_frames_infos().size();
+        for (const auto &streams_frames : net_info.streams_frames_infos()) {
-        for (auto &streams_frames : net_info.streams_frames_infos()) {
+            auto &stream_name_original = streams_frames.stream_name();
-            auto &stream_name = streams_frames.stream_name();
+            auto stream_name = truncate_str(stream_name_original, STREAM_NAME_WIDTH);
             auto stream_direction = (streams_frames.stream_direction() == PROTO__STREAM_DIRECTION__HOST_TO_DEVICE) ? "H2D" : "D2H";
 
             std::string frames;
@@ -109,14 +141,12 @@ size_t MonCommand::print_frames_table(const ProtoMon &mon_message)
             }
             
             std::cout << 
-                std::setw(NETWORK_NAME_WIDTH) << std::left << net_name <<
+                std::setw(STRING_WIDTH) << std::left << net_name <<
-                std::setw(STREAM_NAME_WIDTH) << std::left << stream_name <<
+                std::setw(STRING_WIDTH) << std::left << stream_name <<
                 std::setw(NUMBER_WIDTH) << std::left << stream_direction << 
                 std::setw(NUMBER_WIDTH) << std::left << frames << "\n";
         }
     }
-    
-    return table_lines_count;
 }
 
 #if defined(__GNUC__)
@@ -133,17 +163,49 @@ Expected<uint16_t> get_terminal_line_width()
     return terminal_line_width;
 }
 
-hailo_status MonCommand::print_table()
+void MonCommand::print_tables(const std::vector<ProtoMon> &mon_messages, uint32_t terminal_line_width)
 {
+    print_devices_info_header();
+    for (const auto &mon_message : mon_messages) {
+        print_devices_info_table(mon_message);
+    }
+
+    std::cout << std::string(terminal_line_width, ' ') << "\n";
+    std::cout << std::string(terminal_line_width, ' ') << "\n";   
+    
+    print_networks_info_header();
+
+    for (const auto &mon_message : mon_messages) {
+        print_networks_info_table(mon_message);
+    }
+
+    std::cout << std::string(terminal_line_width, ' ') << "\n";
+    std::cout << std::string(terminal_line_width, ' ') << "\n";
+
+    print_frames_header();
+    for (const auto &mon_message : mon_messages) {
+        print_frames_table(mon_message);
+    }
+}
+
+static volatile bool keep_running = true;
+void signit_handler(int /*dummy*/)
+{
+    keep_running = false;
+}
+
+hailo_status MonCommand::run_monitor()
+{
+    // Note: There is no need to unregister to previous SIGINT handler since we finish running after it is called.
+    signal(SIGINT, signit_handler);
+
     std::chrono::milliseconds time_interval = DEFAULT_SCHEDULER_MON_INTERVAL + EPSILON_TIME;
     auto terminal_line_width_expected = get_terminal_line_width();
     CHECK_EXPECTED_AS_STATUS(terminal_line_width_expected);
     auto terminal_line_width = terminal_line_width_expected.release();
 
-    size_t last_run_total_lines_count = 0;
+    AlternativeTerminal alt_terminal;
-    bool data_was_printed = false;
+    while (keep_running) {
-    while (true) {
-        size_t total_lines_count = 0;
         bool print_warning_msg = true; // Will change to false only if mon directory is valid and there are updated files in it.
 
         auto mon_dir_valid = Filesystem::is_directory(SCHEDULER_MON_TMP_DIR);
@@ -160,14 +222,12 @@ hailo_status MonCommand::print_table()
                 auto file = LockedFile::create(mon_file, "r");
                 if (HAILO_SUCCESS != file.status()) {
                     LOGGER__ERROR("Failed to open and lock file {}, with status: {}", mon_file, file.status());
-                    total_lines_count++;
                     continue;
                 }
 
                 ProtoMon mon_message;
                 if (!mon_message.ParseFromFileDescriptor(file->get_fd())) {
                     LOGGER__WARNING("Failed to ParseFromFileDescriptor monitor file {} with errno {}", mon_file, errno);
-                    total_lines_count++;
                     continue;
                 }
 
@@ -175,38 +235,13 @@ hailo_status MonCommand::print_table()
             }
         }
 
-        total_lines_count += print_networks_info_header();
+        print_tables(mon_messages, terminal_line_width);
-        for (auto &mon_message : mon_messages) {
-            total_lines_count += print_networks_info_table(mon_message);
-        }
-
-        std::cout << std::string(terminal_line_width, ' ') << "\n";
-        std::cout << std::string(terminal_line_width, ' ') << "\n";
-        total_lines_count += 2;
-
-        total_lines_count += print_frames_header();
-        for (auto &mon_message : mon_messages) {
-            total_lines_count += print_frames_table(mon_message);
-        }
-
         if (print_warning_msg) {
-            std::cout << "Monitor did not retrieve any files. This occurs when there is no application currently running. If this is not the case, verify that environment variable '" <<
+            std::cout << FORMAT_GREEN_PRINT << "Monitor did not retrieve any files. This occurs when there is no application currently running.\n"
-                SCHEDULER_MON_ENV_VAR << "' is set to 1.\n";
+            << "If this is not the case, verify that environment variable '" << SCHEDULER_MON_ENV_VAR << "' is set to 1.\n" << FORMAT_NORMAL_PRINT;
-            total_lines_count++;
-
-            if (data_was_printed) {
-                auto lines_to_clear = last_run_total_lines_count - total_lines_count;
-                CliCommon::clear_lines_down(lines_to_clear);
-                total_lines_count += lines_to_clear;
-                data_was_printed = false;
-            }
-        }
-        else {
-            data_was_printed = true;
-            last_run_total_lines_count = total_lines_count;
         }
 
-        CliCommon::reset_cursor(total_lines_count);
+        CliCommon::clear_terminal();
         std::this_thread::sleep_for(DEFAULT_SCHEDULER_MON_INTERVAL);
     }
 

hailort/hailortcli/mon_command.hpp

@@ -11,9 +11,10 @@
 #define _HAILO_MON_COMMAND_HPP_
 
 #include "hailo/hailort.h"
+
 #include "hailortcli.hpp"
 #include "command.hpp"
-#include "scheduler_mon.hpp"
+#include "vdevice/scheduler/scheduler_mon.hpp"
 
 #include "CLI/CLI.hpp"
 
@@ -28,11 +29,15 @@ public:
     virtual hailo_status execute() override;
 
 private:
-    hailo_status print_table();
+    hailo_status run_monitor();
-    size_t print_networks_info_header();
+    void print_tables(const std::vector<ProtoMon> &mon_messages, uint32_t terminal_line_width);
-    size_t print_frames_header();
+    void print_devices_info_header();
-    size_t print_networks_info_table(const ProtoMon &mon_message);
+    void print_networks_info_header();
-    size_t print_frames_table(const ProtoMon &mon_message);
+    void print_frames_header();
+    void print_devices_info_table(const ProtoMon &mon_message);
+    void print_networks_info_table(const ProtoMon &mon_message);
+    void print_frames_table(const ProtoMon &mon_message);
+    hailo_status run_in_alternative_terminal();
 };
 
 } /* namespace hailort */

hailort/hailortcli/parse_hef_command.cpp

@@ -11,60 +11,6 @@
 #include "common/filesystem.hpp"
 #include "hailo/hailort_common.hpp"
 
-#define TAB ("    ")
-
-static std::string add_tabs(uint8_t count)
-{
-    // Each TAB counts as 4 spaces
-    std::string res = "";
-    for (uint8_t i = 0; i < count; i++) {
-        res = res + TAB;
-    }
-    return res;
-}
-
-static std::string get_shape_str(const hailo_stream_info_t &stream_info)
-{
-    switch (stream_info.format.order)
-    {
-    case HAILO_FORMAT_ORDER_HAILO_NMS:
-        return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) +
-            "(number of classes: " + std::to_string(stream_info.nms_info.number_of_classes) +
-            ", max_bboxes_per_class: "+ std::to_string(stream_info.nms_info.max_bboxes_per_class) + ")";
-    case HAILO_FORMAT_ORDER_NC:
-        return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) +
-            "(" + std::to_string(stream_info.hw_shape.features) + ")";
-    case HAILO_FORMAT_ORDER_NHW:
-        return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) +
-            "(" + std::to_string(stream_info.hw_shape.height) + "x" + std::to_string(stream_info.hw_shape.width) + ")";
-    default:
-        return HailoRTCommon::get_format_type_str(stream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(stream_info.format.order) +
-            "(" + std::to_string(stream_info.hw_shape.height) + "x" + std::to_string(stream_info.hw_shape.width) +
-            "x" + std::to_string(stream_info.hw_shape.features) + ")";
-    }
-}
-
-static std::string get_shape_str(const hailo_vstream_info_t &vstream_info)
-{
-    switch (vstream_info.format.order)
-    {
-    case HAILO_FORMAT_ORDER_HAILO_NMS:
-        return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) +
-            "(number of classes: " + std::to_string(vstream_info.nms_shape.number_of_classes) +
-            ", max_bboxes_per_class: " + std::to_string(vstream_info.nms_shape.max_bboxes_per_class) + ")";
-    case HAILO_FORMAT_ORDER_NC:
-        return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) +
-            "(" + std::to_string(vstream_info.shape.features) + ")";
-    case HAILO_FORMAT_ORDER_NHW:
-        return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) +
-            "(" +std::to_string(vstream_info.shape.height) + "x" + std::to_string(vstream_info.shape.width) + ")";
-    default:
-        return HailoRTCommon::get_format_type_str(vstream_info.format.type) + ", " + HailoRTCommon::get_format_order_str(vstream_info.format.order) +
-            "(" + std::to_string(vstream_info.shape.height) + "x" + std::to_string(vstream_info.shape.width) + "x" +
-            std::to_string(vstream_info.shape.features) + ")";
-    }
-}
-
 ParseHefCommand::ParseHefCommand(CLI::App &parent_app) :
     Command(parent_app.add_subcommand("parse-hef", "Parse HEF to get information about its components"))
 {
@@ -94,48 +40,9 @@ hailo_status ParseHefCommand::parse_hefs_info(const std::string &hef_path, bool
     CHECK_EXPECTED_AS_STATUS(hef_exp, "Failed to parse HEF");
     auto hef = hef_exp.release();
 
-    auto network_group_infos = hef.get_network_groups_infos();
+    auto hef_info = hef.get_hef_description(stream_infos, vstream_infos);
-    CHECK_EXPECTED_AS_STATUS(network_group_infos);
+    CHECK_EXPECTED_AS_STATUS(hef_info, "Failed to parse HEF");
-    for (auto &network_group_info : network_group_infos.release()) {
+    std::cout << hef_info.release();
-        auto contexts_str = (network_group_info.is_multi_context ? "Multi Context" : "Single Context");
-        std::cout << "Network group name: " << network_group_info.name << " (" << contexts_str << ")" << std::endl;
-        auto network_infos = hef.get_network_infos(network_group_info.name);
-        CHECK_EXPECTED_AS_STATUS(network_infos, "Failed to parse networks infos");
-        for (auto &network_info : network_infos.value()) {
-            std::cout << add_tabs(1) << "Network name: " << network_info.name << std::endl;
-            if (stream_infos) {
-                std::cout << add_tabs(2) << "Stream infos:" << std::endl;
-                auto input_stream_infos = hef.get_input_stream_infos(network_info.name);
-                CHECK_EXPECTED_AS_STATUS(input_stream_infos, "Failed to parse input stream infos");
-                for (auto &stream_info : input_stream_infos.value()) {
-                    auto shape_str = get_shape_str(stream_info);
-                    std::cout << add_tabs(3) << "Input  " << stream_info.name << " " << shape_str << std::endl;
-                }
-                auto output_stream_infos = hef.get_output_stream_infos(network_info.name);
-                CHECK_EXPECTED_AS_STATUS(output_stream_infos, "Failed to parse output stream infos");
-                for (auto &stream_info : output_stream_infos.value()) {
-                    auto shape_str = get_shape_str(stream_info);
-                    std::cout << add_tabs(3) << "Output " << stream_info.name << " " << shape_str << std::endl;
-                }
-            }
-            if (vstream_infos) {
-                std::cout << add_tabs(2) << "VStream infos:" << std::endl;
-                auto input_vstream_infos = hef.get_input_vstream_infos(network_info.name);
-                CHECK_EXPECTED_AS_STATUS(input_vstream_infos, "Failed to parse input vstream infos");
-                for (auto &vstream_info : input_vstream_infos.value()) {
-                    auto shape_str = get_shape_str(vstream_info);
-                    std::cout << add_tabs(3) << "Input  " << vstream_info.name << " " << shape_str << std::endl;
-                }
-                auto output_vstream_infos = hef.get_output_vstream_infos(network_info.name);
-                CHECK_EXPECTED_AS_STATUS(output_vstream_infos, "Failed to parse output vstream infos");
-                for (auto &vstream_info : output_vstream_infos.value()) {
-                    auto shape_str = get_shape_str(vstream_info);
-                    std::cout << add_tabs(3) << "Output " << vstream_info.name << " " << shape_str << std::endl;
-                }
-            }
-        }
-    }
-    std::cout << std::endl;
     return HAILO_SUCCESS;
 }
 

hailort/hailortcli/run2/live_printer.cpp

@@ -9,15 +9,20 @@
 
 #include "live_printer.hpp"
 #include "../common.hpp"
+#include "common/os_utils.hpp"
+#include "common/utils.hpp"
 #include <sstream>
 #include <iostream>
+
 using namespace hailort;
 
 LivePrinter::LivePrinter(std::chrono::milliseconds interval) :
     m_interval(interval),
     m_stop_event(Event::create_shared(Event::State::not_signalled)),
     m_tracks(),
-    m_mutex()
+    m_mutex(),
+    m_prev_count(0),
+    m_enable_ansi_escape_sequences(CursorAdjustment())
 {
 }
 
@@ -27,43 +32,55 @@ LivePrinter::~LivePrinter()
     if (m_thread.joinable()) {
         m_thread.join();
     }
-    print(false);
+    print();
 }
 
-void LivePrinter::add(std::shared_ptr<Track> track)
+void LivePrinter::add(std::shared_ptr<Track> track, uint8_t level)
 {
     std::unique_lock<std::mutex> lock(m_mutex);
-    m_tracks.emplace_back(track);
+    if (!contains(m_tracks, level)) {
+        m_tracks[level] = {};
+    }
+    m_tracks[level].emplace_back(track);
 }
 
-void LivePrinter::print(bool reset)
+void LivePrinter::print()
 {
     std::stringstream ss;
     uint32_t count = 0;
 
     {
         std::unique_lock<std::mutex> lock(m_mutex);
-        for (auto &track : m_tracks) {
+        for (auto &level_pair : m_tracks) {
+            for (auto &track : level_pair.second) {
                 count += track->get_text(ss);
             }
         }
-
-    std::cout << ss.str() << std::flush;
-    if (reset) {
-        CliCommon::reset_cursor(count);
-        //TODO: what aout leftovers from prev line?
     }
+    CliCommon::reset_cursor(m_prev_count); 
+    // On the first print m_prev_count = 0, so no lines will be deleted
+    std::cout << ss.str() << std::flush;
+    m_prev_count = count;
 }
 
-void LivePrinter::start()
+hailo_status LivePrinter::start()
 {
+    for (auto &level_pair : m_tracks) {
+        for (auto &track : level_pair.second) {
+            CHECK_SUCCESS(track->start());
+        }
+    }
+    
     m_thread = std::thread([this] () {
+        OsUtils::set_current_thread_name("LIVE_PRINTER");
         while (true) {
-            print(true);
+            print();
             auto status = m_stop_event->wait(m_interval);
             if (HAILO_TIMEOUT != status) {
                 break;
             }
         }
     });
+
+    return HAILO_SUCCESS;
 }

hailort/hailortcli/run2/live_printer.hpp

@@ -10,12 +10,14 @@
 #ifndef _HAILO_HAILORTCLI_RUN2_LIVE_PRINTER_HPP_
 #define _HAILO_HAILORTCLI_RUN2_LIVE_PRINTER_HPP_
 
+#include "common/os_utils.hpp"
 #include "hailo/event.hpp"
 #include <stdint.h>
 #include <chrono>
 #include <mutex>
 #include <thread>
 #include <atomic>
+#include <map>
 
 class LivePrinter final
 {
@@ -23,21 +25,30 @@ public:
     class Track
     {
     public:
+        Track() : m_started(false)
+        {}
+
+        virtual hailo_status start() = 0;
         virtual uint32_t get_text(std::stringstream &ss) = 0;
+
+    protected:
+        bool m_started;
     };
 
     LivePrinter(std::chrono::milliseconds interval);
     ~LivePrinter();
-    void add(std::shared_ptr<Track> track);
+    void add(std::shared_ptr<Track> track, uint8_t level); // prints tracks in consecutive order from low-to-high levels
-    void print(bool reset);
+    void print();
-    void start();
+    hailo_status start();
 
 private:
     std::chrono::milliseconds m_interval;
     hailort::EventPtr m_stop_event;
-    std::vector<std::shared_ptr<Track>> m_tracks;
+    std::map<uint8_t, std::vector<std::shared_ptr<Track>>> m_tracks;
     std::thread m_thread;
     std::mutex m_mutex;
+    uint32_t m_prev_count;
+    hailort::CursorAdjustment m_enable_ansi_escape_sequences;
 };
 
 #endif /* _HAILO_HAILORTCLI_RUN2_LIVE_PRINTER_HPP_ */

hailort/hailortcli/run2/measurement_live_track.cpp

@@ -0,0 +1,141 @@
+/**
+ * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file measurement_live_track.cpp
+ * @brief Device measurements live track
+ **/
+
+#include "hailo/hailort.h"
+
+#include "common/device_measurements.hpp"
+#include "common/utils.hpp"
+
+#include "measurement_live_track.hpp"
+
+#include <spdlog/fmt/fmt.h>
+#include <sstream>
+
+
+using namespace hailort;
+
+Expected<std::shared_ptr<MeasurementLiveTrack>> MeasurementLiveTrack::create_shared(Device &device, bool measure_power, bool measure_current,
+    bool measure_temp)
+{
+    std::shared_ptr<PowerMeasurement> power_measurement = nullptr;
+    if (measure_power) {
+        auto power_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__POWER);
+        CHECK_EXPECTED(power_measurement_exp);
+        power_measurement = power_measurement_exp.release();
+    }
+
+    std::shared_ptr<PowerMeasurement> current_measurement = nullptr;
+    if (measure_current) {
+        auto current_measurement_exp = PowerMeasurement::create_shared(device, HAILO_POWER_MEASUREMENT_TYPES__CURRENT);
+        CHECK_EXPECTED(current_measurement_exp);
+        current_measurement = current_measurement_exp.release();
+    }
+
+    std::shared_ptr<TemperatureMeasurement> temp_measurement = nullptr;
+    if (measure_temp) {
+        auto temp_measurement_exp = TemperatureMeasurement::create_shared(device);
+        CHECK_EXPECTED(temp_measurement_exp);
+        temp_measurement = temp_measurement_exp.release();
+    }
+
+    auto ptr = make_shared_nothrow<MeasurementLiveTrack>(power_measurement, current_measurement, temp_measurement, device.get_dev_id());
+    CHECK_NOT_NULL_AS_EXPECTED(ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+    return ptr;
+}
+
+MeasurementLiveTrack::MeasurementLiveTrack(std::shared_ptr<PowerMeasurement> power_measurement,
+    std::shared_ptr<PowerMeasurement> current_measurement, std::shared_ptr<TemperatureMeasurement> temp_measurement,
+    const std::string &device_id) :
+        LivePrinter::Track(), m_power_measurement(std::move(power_measurement)), m_current_measurement(std::move(current_measurement)),
+        m_temp_measurement(std::move(temp_measurement)), m_device_id(device_id)
+{}
+
+hailo_status MeasurementLiveTrack::start()
+{
+    if (m_power_measurement) {
+        CHECK_SUCCESS(m_power_measurement->start_measurement());
+    }
+
+    if (m_current_measurement) {
+        CHECK_SUCCESS(m_current_measurement->start_measurement());
+    }
+
+    if (m_temp_measurement) {
+        CHECK_SUCCESS(m_temp_measurement->start_measurement());
+    }
+
+    m_started = true;
+
+    return HAILO_SUCCESS;
+}
+
+uint32_t MeasurementLiveTrack::get_text(std::stringstream &ss)
+{
+    if (!m_started) {
+        return 0;
+    }
+
+    auto rows_count = 0;
+
+    if (m_power_measurement || m_current_measurement || m_temp_measurement) {
+        ss << fmt::format("\nMeasurements for device {}\n", m_device_id);
+        rows_count += 2;
+    }
+
+    if (m_power_measurement) {
+        auto measurement_info = m_power_measurement->get_data();
+        if (auto min = measurement_info.min()) {
+            ss << fmt::format("\tMinimum power consumption: {:.2f} {}\n", *min, m_power_measurement->measurement_unit());
+            rows_count++;
+        }
+        if (auto mean = measurement_info.mean()) {
+            ss << fmt::format("\tAverage power consumption: {:.2f} {}\n", *mean, m_power_measurement->measurement_unit());
+            rows_count++;
+        }
+        if (auto max = measurement_info.max()) {
+            ss << fmt::format("\tMaximum power consumption: {:.2f} {}\n", *max, m_power_measurement->measurement_unit());
+            rows_count++;
+        }
+    }
+
+    if (m_current_measurement) {
+        auto measurement_info = m_current_measurement->get_data();
+        if (auto min = measurement_info.min()) {
+            ss << fmt::format("\tMinimum current consumption: {:.2f} {}\n", *min, m_current_measurement->measurement_unit());
+            rows_count++;
+        }
+        if (auto mean = measurement_info.mean()) {
+            ss << fmt::format("\tAverage current consumption: {:.2f} {}\n", *mean, m_current_measurement->measurement_unit());
+            rows_count++;
+        }
+        if (auto max = measurement_info.max()) {
+            ss << fmt::format("\tMaximum current consumption: {:.2f} {}\n", *max, m_current_measurement->measurement_unit());
+            rows_count++;
+        }
+    }
+
+    if (m_temp_measurement) {
+        auto measurement_info = m_temp_measurement->get_data();
+        if (auto min = measurement_info.min()) {
+            ss << fmt::format("\tMinimum chip temperature: {:.2f} {}\n", *min, m_temp_measurement->measurement_unit());
+            rows_count++;
+        }
+        if (auto mean = measurement_info.mean()) {
+            ss << fmt::format("\tAverage chip temperature: {:.2f} {}\n", *mean, m_temp_measurement->measurement_unit());
+            rows_count++;
+        }
+        if (auto max = measurement_info.max()) {
+            ss << fmt::format("\tMaximum chip temperature: {:.2f} {}\n", *max, m_temp_measurement->measurement_unit());
+            rows_count++;
+        }
+    }
+
+    return rows_count;
+}

hailort/hailortcli/run2/measurement_live_track.hpp

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
+ * Distributed under the MIT license (https://opensource.org/licenses/MIT)
+ **/
+/**
+ * @file measurement_live_track.hpp
+ * @brief Device measurements live track
+ **/
+
+#ifndef _HAILO_HAILORTCLI_RUN2_MEASUREMENT_LIVE_TRACK_HPP_
+#define _HAILO_HAILORTCLI_RUN2_MEASUREMENT_LIVE_TRACK_HPP_
+
+#include "hailo/hailort.h"
+
+#include "common/device_measurements.hpp"
+
+#include "live_printer.hpp"
+
+
+class MeasurementLiveTrack : public LivePrinter::Track
+{
+public:
+    static hailort::Expected<std::shared_ptr<MeasurementLiveTrack>> create_shared(hailort::Device &vdevice, bool measure_power,
+        bool measure_current, bool measure_temp);
+
+    virtual ~MeasurementLiveTrack() = default;
+    virtual hailo_status start() override;
+    virtual uint32_t get_text(std::stringstream &ss) override;
+
+    MeasurementLiveTrack(std::shared_ptr<PowerMeasurement> power_measurement, std::shared_ptr<PowerMeasurement> current_measurement,
+        std::shared_ptr<TemperatureMeasurement> temp_measurement, const std::string &device_id);
+
+private:
+    std::shared_ptr<PowerMeasurement> m_power_measurement;
+    std::shared_ptr<PowerMeasurement> m_current_measurement;
+    std::shared_ptr<TemperatureMeasurement> m_temp_measurement;
+    
+    std::string m_device_id;
+};
+
+#endif /* _HAILO_HAILORTCLI_RUN2_MEASUREMENT_LIVE_TRACK_HPP_ */

hailort/hailortcli/run2/network_live_track.cpp

@@ -8,26 +8,64 @@
  **/
 
 #include "network_live_track.hpp"
+#include "../infer_stats_printer.hpp"
+
 #include <spdlog/fmt/fmt.h>
 #include <sstream>
 
-NetworkLiveTrack::NetworkLiveTrack(const std::string &name) :
+NetworkLiveTrack::NetworkLiveTrack(const std::string &name, std::shared_ptr<ConfiguredNetworkGroup> cng, LatencyMeterPtr overall_latency_meter) :
-    m_name(name), m_count(0), m_last_get_time(std::chrono::steady_clock::now())
+    m_name(name), m_count(0), m_last_get_time(), m_cng(cng), m_overall_latency_meter(overall_latency_meter)
 {
 }
 
+hailo_status NetworkLiveTrack::start()
+{
+    m_last_get_time = std::chrono::steady_clock::now();
+    m_count = 0;
+    m_started = true;
+
+    return HAILO_SUCCESS;
+}
+
 uint32_t NetworkLiveTrack::get_text(std::stringstream &ss)
 {
+    if (!m_started) {
+        return 0;
+    }
+
     auto elapsed_time = std::chrono::steady_clock::now() - m_last_get_time;
     auto count = m_count.load();
 
     auto fps = count / std::chrono::duration<double>(elapsed_time).count();
-    ss << fmt::format("{} - fps: {:.2f}\n", m_name, fps);
+    ss << fmt::format("{}:\n\t|  fps: {:.2f}", m_name, fps);
 
-    return 1;
+    auto hw_latency_measurement = m_cng->get_latency_measurement();
+    if (hw_latency_measurement) {
+        ss << fmt::format("  |  hw latency: {:.2f} ms", InferResultsFormatUtils::latency_result_to_ms(hw_latency_measurement->avg_hw_latency));
+    }
+    else if (HAILO_NOT_AVAILABLE != hw_latency_measurement.status()) { // HAILO_NOT_AVAILABLE is a valid error, we ignore it
+        ss << fmt::format("  |  hw latency: failed with status={}", hw_latency_measurement.status());
+    }
+
+    if (m_overall_latency_meter) {
+        auto overall_latency_measurement = m_overall_latency_meter->get_latency(true);
+        if (overall_latency_measurement) {
+            ss << fmt::format("  |  overall latency: {:.2f} ms", InferResultsFormatUtils::latency_result_to_ms(*overall_latency_measurement));
+        }
+        else if (HAILO_NOT_AVAILABLE != overall_latency_measurement.status()) { // HAILO_NOT_AVAILABLE is a valid error, we ignore it
+            ss << fmt::format("  |  overall latency: failed with status={}", overall_latency_measurement.status());
+        }
+    }
+    ss << "\n";
+
+    return 2;
 }
 
 void NetworkLiveTrack::progress()
 {
+    if (!m_started) {
+        return;
+    }
+
     m_count++;
 }

hailort/hailortcli/run2/network_live_track.hpp

@@ -7,23 +7,32 @@
  * @brief Network live track
  **/
 
-#include "live_printer.hpp"
-
 #ifndef _HAILO_HAILORTCLI_RUN2_NETWORK_LIVE_TRACK_HPP_
 #define _HAILO_HAILORTCLI_RUN2_NETWORK_LIVE_TRACK_HPP_
 
+#include "hailo/hailort.h"
+#include "hailo/network_group.hpp"
+
+#include "common/latency_meter.hpp"
+
+#include "live_printer.hpp"
+
+
 class NetworkLiveTrack : public LivePrinter::Track
 {
 public:
-    NetworkLiveTrack(const std::string &name);
+    NetworkLiveTrack(const std::string &name, std::shared_ptr<hailort::ConfiguredNetworkGroup> cng, hailort::LatencyMeterPtr overall_latency_meter);
     virtual ~NetworkLiveTrack() = default;
-    uint32_t get_text(std::stringstream &ss);
+    virtual hailo_status start() override;
+    virtual uint32_t get_text(std::stringstream &ss) override;
     void progress();
 
 private:
     std::string m_name;
     std::atomic<uint32_t> m_count;
     std::chrono::time_point<std::chrono::steady_clock> m_last_get_time;
+    std::shared_ptr<hailort::ConfiguredNetworkGroup> m_cng;
+    hailort::LatencyMeterPtr m_overall_latency_meter;
 };
 
 #endif /* _HAILO_HAILORTCLI_RUN2_NETWORK_LIVE_TRACK_HPP_ */

hailort/hailortcli/run2/network_runner.cpp

@@ -7,12 +7,35 @@
  * @brief Run network on hailo device
  **/
 
-#include "network_runner.hpp"
+#include "hailo/hailort.h"
+#include "hailo/hailort_common.hpp"
+#include "hailo/hailort_defaults.hpp"
+
 #include "common/async_thread.hpp"
-#include "hailort_defaults.hpp" //TODO: not API
+#include "common/file_utils.hpp"
+#include "common/latency_meter.hpp"
+
+#include "network_runner.hpp"
+
 
 using namespace hailort;
 
+
+class SignalEventScopeGuard final
+{
+public:
+    SignalEventScopeGuard(Event &event) : m_event(event)
+    {}
+
+    ~SignalEventScopeGuard()
+    {
+        m_event.signal();
+    }
+
+    Event &m_event;
+};
+
+
 //TODO: duplicated
 static hailo_status wait_for_threads(std::vector<AsyncThreadPtr<hailo_status>> &threads)
 {
@@ -31,10 +54,17 @@ VStreamParams::VStreamParams() : name(), params(HailoRTDefaults::get_vstreams_pa
 {
 }
 
+NetworkParams::NetworkParams() : hef_path(), net_group_name(), vstream_params(), scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN),
+    batch_size(HAILO_DEFAULT_BATCH_SIZE), scheduler_threshold(0), scheduler_timeout_ms(0), framerate(UNLIMITED_FRAMERATE), measure_hw_latency(false),
+    measure_overall_latency(false)
+{
+}
+
 NetworkRunner::NetworkRunner(const NetworkParams &params, const std::string &name,
-    std::vector<InputVStream> &&input_vstreams, std::vector<OutputVStream> &&output_vstreams)
+    std::vector<InputVStream> &&input_vstreams, std::vector<OutputVStream> &&output_vstreams,
+    std::shared_ptr<ConfiguredNetworkGroup> cng, LatencyMeterPtr overall_latency_meter)
     : m_params(params), m_name(name), m_input_vstreams(std::move(input_vstreams)),
-      m_output_vstreams(std::move(output_vstreams))
+      m_output_vstreams(std::move(output_vstreams)), m_cng(cng), m_overall_latency_meter(overall_latency_meter)
 {
 }
 
@@ -51,19 +81,22 @@ Expected<std::shared_ptr<NetworkRunner>> NetworkRunner::create_shared(VDevice &v
         net_group_name = net_groups_names[0];
     }
 
-    auto interface = vdevice.get_default_streams_interface();
+    auto cfg_params = vdevice.create_configure_params(hef.value(), net_group_name);
-    CHECK_EXPECTED(interface, "Failed to get default streams interface");
-
-    auto cfg_params = hef->create_configure_params(*interface, net_group_name);
     CHECK_EXPECTED(cfg_params);
     cfg_params->batch_size = params.batch_size;
+    if (params.measure_hw_latency) {
+        cfg_params->latency |= HAILO_LATENCY_MEASURE;
+    }
     auto cfgr_net_groups = vdevice.configure(hef.value(), {{net_group_name, cfg_params.value()}});
     CHECK_EXPECTED(cfgr_net_groups);
     assert(1 == cfgr_net_groups->size());
     auto cfgr_net_group = cfgr_net_groups.value()[0];
 
+    if (HAILO_SCHEDULING_ALGORITHM_NONE!= params.scheduling_algorithm) {
         CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_threshold(params.scheduler_threshold));
         CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_timeout(std::chrono::milliseconds(params.scheduler_timeout_ms)));
+        CHECK_SUCCESS_AS_EXPECTED(cfgr_net_group->set_scheduler_priority(params.scheduler_priority));
+    }
 
     std::map<std::string, hailo_vstream_params_t> vstreams_params;
     for (auto &vstream_params : params.vstream_params) {
@@ -72,23 +105,69 @@ Expected<std::shared_ptr<NetworkRunner>> NetworkRunner::create_shared(VDevice &v
     auto vstreams = create_vstreams(*cfgr_net_group, vstreams_params);
     CHECK_EXPECTED(vstreams);
 
-    auto net_runner = make_shared_nothrow<NetworkRunner>(params, net_group_name, std::move(vstreams->first), std::move(vstreams->second));
+    LatencyMeterPtr overall_latency_meter = nullptr;
+    if (params.measure_overall_latency) {
+        CHECK_AS_EXPECTED((1 == vstreams->first.size()), HAILO_INVALID_OPERATION,
+            "Overall latency measurement over multiple inputs network is not supported");
+
+        std::set<std::string> output_names;
+        for (auto &output_vstream : vstreams->second) {
+            output_names.insert(output_vstream.name());
+        }
+
+        overall_latency_meter = make_shared_nothrow<LatencyMeter>(output_names, OVERALL_LATENCY_TIMESTAMPS_LIST_LENGTH);
+        CHECK_NOT_NULL_AS_EXPECTED(overall_latency_meter, HAILO_OUT_OF_HOST_MEMORY);
+    }
+    auto net_runner = make_shared_nothrow<NetworkRunner>(params, net_group_name, std::move(vstreams->first),
+        std::move(vstreams->second), cfgr_net_group, overall_latency_meter);
     CHECK_NOT_NULL_AS_EXPECTED(net_runner, HAILO_OUT_OF_HOST_MEMORY);
     return net_runner;
 }
 
-hailo_status NetworkRunner::run_input_vstream(InputVStream &vstream)
+Expected<BufferPtr> NetworkRunner::create_dataset_from_input_file(const std::string &file_path,
+    const InputVStream &input_vstream)
 {
-    auto dataset = Buffer::create(vstream.get_frame_size(), 0xAB);
+    auto buffer = read_binary_file(file_path);
-    CHECK_EXPECTED_AS_STATUS(dataset);
+    CHECK_EXPECTED(buffer);
+    CHECK_AS_EXPECTED(0 == (buffer->size() % input_vstream.get_frame_size()), HAILO_INVALID_ARGUMENT,
+        "Input file ({}) size {} must be a multiple of the frame size {} ({})",
+        file_path, buffer->size(), input_vstream.get_frame_size(), input_vstream.name());
+
+    auto buffer_ptr = make_shared_nothrow<Buffer>(buffer.release());
+    CHECK_NOT_NULL_AS_EXPECTED(buffer_ptr, HAILO_OUT_OF_HOST_MEMORY);
+
+    return buffer_ptr;
+}
+
+
+Expected<BufferPtr> NetworkRunner::create_constant_dataset(const InputVStream &input_vstream)
+{
+    const uint8_t const_byte = 0xAB;
+    auto constant_buffer = Buffer::create_shared(input_vstream.get_frame_size(), const_byte);
+    CHECK_EXPECTED(constant_buffer);
+
+    return constant_buffer.release();
+}
+
+hailo_status NetworkRunner::run_input_vstream(InputVStream &vstream, Event &shutdown_event, BufferPtr dataset,
+    LatencyMeterPtr overall_latency_meter)
+{
+    auto signal_event_scope_guard = SignalEventScopeGuard(shutdown_event);
+
     auto last_write_time = std::chrono::steady_clock::now();
     auto framerate_interval = std::chrono::duration<double>(1) / m_params.framerate;
+    size_t buffer_offset = 0;
     while(true) {
-        auto status = vstream.write(MemoryView(dataset.value()));
+        if (overall_latency_meter) {
+            overall_latency_meter->add_start_sample(std::chrono::steady_clock::now().time_since_epoch());
+        }
+        auto status = vstream.write(MemoryView((dataset->data() + buffer_offset), vstream.get_frame_size()));
         if (status == HAILO_STREAM_ABORTED_BY_USER) {
             return status;
         }
         CHECK_SUCCESS(status);
+        buffer_offset += vstream.get_frame_size();
+        buffer_offset %= dataset->size();
 
         if (m_params.framerate != UNLIMITED_FRAMERATE) {
             auto elapsed_time = std::chrono::steady_clock::now() - last_write_time;
@@ -99,8 +178,11 @@ hailo_status NetworkRunner::run_input_vstream(InputVStream &vstream)
     return HAILO_SUCCESS;
 }
 
-hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool first, std::shared_ptr<NetworkLiveTrack> net_live_track)
+hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool first, std::shared_ptr<NetworkLiveTrack> net_live_track,
+    Event &shutdown_event, LatencyMeterPtr overall_latency_meter)
 {
+    auto signal_event_scope_guard = SignalEventScopeGuard(shutdown_event);
+
     auto result = Buffer::create(vstream.get_frame_size());
     CHECK_EXPECTED_AS_STATUS(result);
     while(true) {
@@ -109,6 +191,9 @@ hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool firs
             return status;
         }
         CHECK_SUCCESS(status);
+        if (overall_latency_meter) {
+            overall_latency_meter->add_end_sample(vstream.name(), std::chrono::steady_clock::now().time_since_epoch());
+        }
         if (first) {
             net_live_track->progress();
         }
@@ -116,27 +201,54 @@ hailo_status NetworkRunner::run_output_vstream(OutputVStream &vstream, bool firs
     return HAILO_SUCCESS;
 }
 
-hailo_status NetworkRunner::run(Event &shutdown_event, LivePrinter &live_printer)
+hailo_status NetworkRunner::run(Event &shutdown_event, LivePrinter &live_printer, Barrier &barrier)
 {
+    auto ang = std::unique_ptr<ActivatedNetworkGroup>(nullptr);
+    if (HAILO_SCHEDULING_ALGORITHM_NONE == m_params.scheduling_algorithm) {
+        auto ang_exp = m_cng->activate();
+        if (!ang_exp) {
+            barrier.terminate();
+        }
+        CHECK_EXPECTED_AS_STATUS(ang_exp);
+        ang = ang_exp.release();
+    }
+
+    auto net_live_track = std::make_shared<NetworkLiveTrack>(m_name, m_cng, m_overall_latency_meter);
+    live_printer.add(net_live_track, 1); //support progress over multiple outputs
+    barrier.arrive_and_wait();
+
     std::vector<AsyncThreadPtr<hailo_status>> threads;
     for (auto &input_vstream : m_input_vstreams) {
-        threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>([this, &input_vstream](){
+        BufferPtr dataset = nullptr;
-            return run_input_vstream(input_vstream);
+        for (auto &params : m_params.vstream_params) {
+            if ((input_vstream.name() == params.name) && (!params.input_file_path.empty())) {
+                auto dataset_exp = create_dataset_from_input_file(params.input_file_path, input_vstream);
+                CHECK_EXPECTED_AS_STATUS(dataset_exp);
+                dataset = dataset_exp.release();
+            }
+        }
+        if (nullptr == dataset) {
+            auto dataset_exp = create_constant_dataset(input_vstream);
+            CHECK_EXPECTED_AS_STATUS(dataset_exp);
+            dataset = dataset_exp.release();
+        }
+
+        threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("SEND", [this, &input_vstream, &shutdown_event,
+            dataset](){
+            return run_input_vstream(input_vstream, shutdown_event, dataset, m_overall_latency_meter);
         }));
     }
 
-    auto net_live_track = std::make_shared<NetworkLiveTrack>(m_name);
+    bool first = true; //TODO: check with multiple outputs
-    live_printer.add(net_live_track);//support progress over multiple outputs
-
-    bool first = true;//TODO: check with multiple outputs
     for (auto &output_vstream : m_output_vstreams) {
-        threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>([&output_vstream, first, net_live_track](){
+        threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("RECV", [this, &output_vstream, first, net_live_track,
-            return run_output_vstream(output_vstream, first, net_live_track);
+        &shutdown_event](){
+            return run_output_vstream(output_vstream, first, net_live_track, shutdown_event, m_overall_latency_meter);
         }));
         first = false;
     }
 
-    //TODO: signal a barrier that we should start infer and timer. return threads and move stop outside?
+    //TODO: return threads and move stop outside?
     CHECK_SUCCESS(shutdown_event.wait(HAILO_INFINITE_TIMEOUT));
     stop();
     return wait_for_threads(threads);

hailort/hailortcli/run2/network_runner.hpp

@@ -10,11 +10,16 @@
 #ifndef _HAILO_HAILORTCLI_RUN2_NETWORK_RUNNER_HPP_
 #define _HAILO_HAILORTCLI_RUN2_NETWORK_RUNNER_HPP_
 
+#include "common/barrier.hpp"
+
 #include "hailo/vdevice.hpp"
 #include "hailo/vstream.hpp"
 #include "hailo/event.hpp"
 #include "hailo/network_group.hpp"
 #include "hailo/expected.hpp"
+#include "hailo/buffer.hpp"
+
+#include "../hailortcli.hpp"
 
 #include "live_printer.hpp"
 #include "network_live_track.hpp"
@@ -30,43 +35,58 @@ struct VStreamParams
 
     std::string name;
     hailo_vstream_params_t params;
+    std::string input_file_path;
 };
 
 struct NetworkParams
 {
+    NetworkParams();
+
     std::string hef_path;
     std::string net_group_name;
     std::vector<VStreamParams> vstream_params;
+    hailo_scheduling_algorithm_t scheduling_algorithm;
 
     // Network parameters
     uint16_t batch_size;
     uint32_t scheduler_threshold;
     uint32_t scheduler_timeout_ms;
+    uint8_t scheduler_priority;
 
     // Run parameters
     uint32_t framerate;
+
+    bool measure_hw_latency;
+    bool measure_overall_latency;
 };
 
 class NetworkRunner
 {
 public:
     NetworkRunner(const NetworkParams &params, const std::string &name,
-        std::vector<hailort::InputVStream> &&input_vstreams, std::vector<hailort::OutputVStream> &&output_vstreams);
+        std::vector<hailort::InputVStream> &&input_vstreams, std::vector<hailort::OutputVStream> &&output_vstreams,
+        std::shared_ptr<hailort::ConfiguredNetworkGroup> cng, hailort::LatencyMeterPtr overall_latency_meter);
     static hailort::Expected<std::shared_ptr<NetworkRunner>> create_shared(hailort::VDevice &vdevice, const NetworkParams &params);
-    hailo_status run(hailort::Event &shutdown_event, LivePrinter &live_printer);
+    hailo_status run(hailort::Event &shutdown_event, LivePrinter &live_printer, hailort::Barrier &barrier);
     void stop();
 
 private:
     static hailort::Expected<std::pair<std::vector<hailort::InputVStream>, std::vector<hailort::OutputVStream>>> create_vstreams(
         hailort::ConfiguredNetworkGroup &net_group, const std::map<std::string, hailo_vstream_params_t> &params);
-    hailo_status run_input_vstream(hailort::InputVStream &vstream);
+    hailo_status run_input_vstream(hailort::InputVStream &vstream, hailort::Event &shutdown_event, hailort::BufferPtr dataset,
-    static hailo_status run_output_vstream(hailort::OutputVStream &vstream, bool first, std::shared_ptr<NetworkLiveTrack> net_live_track);
+        hailort::LatencyMeterPtr overall_latency_meter);
+    static hailo_status run_output_vstream(hailort::OutputVStream &vstream, bool first, std::shared_ptr<NetworkLiveTrack> net_live_track,
+        hailort::Event &shutdown_event, hailort::LatencyMeterPtr overall_latency_meter);
 
+static hailort::Expected<hailort::BufferPtr> create_constant_dataset(const hailort::InputVStream &input_vstream);
+static hailort::Expected<hailort::BufferPtr> create_dataset_from_input_file(const std::string &file_path, const hailort::InputVStream &input_vstream);
 
     const NetworkParams &m_params;//TODO: copy instead of ref?
     std::string m_name;
     std::vector<hailort::InputVStream> m_input_vstreams;
     std::vector<hailort::OutputVStream> m_output_vstreams;
+    std::shared_ptr<hailort::ConfiguredNetworkGroup> m_cng;
+    hailort::LatencyMeterPtr m_overall_latency_meter;
 };
 
 #endif /* _HAILO_HAILORTCLI_RUN2_NETWORK_RUNNER_HPP_ */

hailort/hailortcli/run2/run2_command.cpp

@@ -10,8 +10,10 @@
 #include "run2_command.hpp"
 #include "live_printer.hpp"
 #include "timer_live_track.hpp"
+#include "measurement_live_track.hpp"
 #include "network_runner.hpp"
 
+#include "common/barrier.hpp"
 #include "common/async_thread.hpp"
 #include "hailo/vdevice.hpp"
 #include "hailo/hef.hpp"
@@ -91,6 +93,10 @@ VStreamApp::VStreamApp(const std::string &description, const std::string &name,
     add_option("name", m_params.name, "vStream name")
         ->check(VStreamNameValidator(hef_path_option, net_group_name_option));
 
+    add_option("--input-file", m_params.input_file_path,
+        "Input file path. If not given, random data will be used. File format should be raw binary data with size that is a factor of the input shape size")
+        ->default_val("");
+
     auto format_opt_group = add_option_group("Format");
     format_opt_group->add_option("--type", m_params.params.user_buffer_format.type, "Format type")
         ->transform(HailoCheckedTransformer<hailo_format_type_t>({
@@ -116,14 +122,16 @@ VStreamApp::VStreamApp(const std::string &description, const std::string &name,
             { "nchw", HAILO_FORMAT_ORDER_NCHW },
             { "yuy2", HAILO_FORMAT_ORDER_YUY2 },
             { "nv12", HAILO_FORMAT_ORDER_NV12 },
-            { "nv21", HAILO_FORMAT_ORDER_NV21 }
+            { "nv21", HAILO_FORMAT_ORDER_NV21 },
+            { "rgb4", HAILO_FORMAT_ORDER_RGB4 },
+            { "i420", HAILO_FORMAT_ORDER_I420 }
         }))
         ->default_val("auto");
 
     add_flag_callback(format_opt_group, "-q,--quantized,!--no-quantized", "Whether or not data is quantized",
         [this](bool result){
             m_params.params.user_buffer_format.flags = result ?
-                m_params.params.user_buffer_format.flags | HAILO_FORMAT_FLAGS_QUANTIZED :
+                static_cast<hailo_format_flags_t>(m_params.params.user_buffer_format.flags | HAILO_FORMAT_FLAGS_QUANTIZED) :
                 static_cast<hailo_format_flags_t>(m_params.params.user_buffer_format.flags & (~HAILO_FORMAT_FLAGS_QUANTIZED));})
         ->run_callback_for_default()
         ->default_val(true); // default_val() must be after run_callback_for_default()
@@ -188,10 +196,14 @@ NetworkApp::NetworkApp(const std::string &description, const std::string &name)
     net_params->add_option("--batch-size", m_params.batch_size, "Batch size")->default_val(HAILO_DEFAULT_BATCH_SIZE);
     net_params->add_option("--scheduler-threshold", m_params.scheduler_threshold, "Scheduler threshold")->default_val(0);
     net_params->add_option("--scheduler-timeout", m_params.scheduler_timeout_ms, "Scheduler timeout in milliseconds")->default_val(0);
+    net_params->add_option("--scheduler-priority", m_params.scheduler_priority, "Scheduler priority")->default_val(HAILO_SCHEDULER_PRIORITY_NORMAL);
 
     auto run_params = add_option_group("Run Parameters");
     run_params->add_option("--framerate", m_params.framerate, "Input vStreams framerate")->default_val(UNLIMITED_FRAMERATE);
 
+    // TODO: support multiple scheduling algorithms
+    m_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN;
+
     add_vstream_app_subcom(hef_path_option, net_group_name_option);
 }
 
@@ -235,19 +247,74 @@ public:
 
     const std::vector<NetworkParams>& get_network_params();
     std::chrono::seconds get_time_to_run();
+    std::vector<hailo_device_id_t> get_dev_ids();
+    uint32_t get_device_count();
+    bool get_measure_power();
+    bool get_measure_current();
+    bool get_measure_temp();
+    bool get_multi_process_service();
+    const std::string &get_group_id();
+
+    void set_scheduling_algorithm(hailo_scheduling_algorithm_t scheduling_algorithm);
+    void set_measure_latency();
 
 private:
     void add_net_app_subcom();
     std::vector<NetworkParams> m_network_params;
     uint32_t m_time_to_run;
+    std::vector<std::string> m_device_id;
+    uint32_t m_device_count;
+    bool m_multi_process_service;
+    std::string m_group_id;
+
+    bool m_measure_hw_latency;
+    bool m_measure_overall_latency;
+
+    bool m_measure_power;
+    bool m_measure_current;
+    bool m_measure_temp;
 };
 
+
 Run2::Run2() : CLI::App("Run networks (preview)", "run2")
 {
     add_net_app_subcom();
     add_option("-t,--time-to-run", m_time_to_run, "Time to run (seconds)")
         ->default_val(DEFAULT_TIME_TO_RUN_SECONDS)
         ->check(CLI::PositiveNumber);
+
+    auto vdevice_options_group = add_option_group("VDevice Options");
+
+    auto dev_id_opt = vdevice_options_group->add_option("-s,--device-id", m_device_id,
+        "Device id, same as returned from `hailortcli scan` command. For multiple devices, use space as separator.");
+
+    vdevice_options_group->add_option("--device-count", m_device_count, "VDevice device count")
+        ->default_val(HAILO_DEFAULT_DEVICE_COUNT)
+        ->check(CLI::PositiveNumber)
+        ->excludes(dev_id_opt);
+
+    vdevice_options_group->add_flag("--multi-process-service", m_multi_process_service, "VDevice multi process service")
+        ->default_val(false);
+
+    vdevice_options_group->add_option("--group-id", m_group_id, "VDevice group id")
+        ->default_val(HAILO_DEFAULT_VDEVICE_GROUP_ID);
+
+    auto measurement_options_group = add_option_group("Measurement Options");
+
+    auto measure_power_opt = measurement_options_group->add_flag("--measure-power", m_measure_power, "Measure power consumption")
+        ->default_val(false);
+    
+    measurement_options_group->add_flag("--measure-current", m_measure_current, "Measure current")->excludes(measure_power_opt)
+        ->default_val(false);
+
+    measurement_options_group->add_flag("--measure-latency", m_measure_hw_latency, "Measure network latency")
+        ->default_val(false);
+    
+    measurement_options_group->add_flag("--measure-overall-latency", m_measure_overall_latency, "Measure overall latency measurement")
+        ->default_val(false);
+
+    measurement_options_group->add_flag("--measure-temp", m_measure_temp, "Measure chip temperature")
+        ->default_val(false);
 }
 
 void Run2::add_net_app_subcom()
@@ -286,6 +353,65 @@ std::chrono::seconds Run2::get_time_to_run()
     return std::chrono::seconds(m_time_to_run);
 }
 
+bool Run2::get_measure_power()
+{
+    return m_measure_power;
+}
+
+bool Run2::get_measure_current()
+{
+    return m_measure_current;
+}
+
+bool Run2::get_measure_temp()
+{
+    return m_measure_temp;
+}
+
+std::vector<hailo_device_id_t> Run2::get_dev_ids()
+{
+    std::vector<hailo_device_id_t> res;
+    res.reserve(m_device_id.size());
+    for (auto &id_str : m_device_id) {
+        hailo_device_id_t id = {};
+        std::memset(id.id, 0, sizeof(id.id));
+        std::strncpy(id.id, id_str.c_str(), sizeof(id.id) - 1);
+        res.push_back(id);
+    }
+    return res;
+}
+
+uint32_t Run2::get_device_count()
+{
+    return m_device_count;
+}
+
+void Run2::set_scheduling_algorithm(hailo_scheduling_algorithm_t scheduling_algorithm)
+{
+    for (auto &params: m_network_params) {
+        params.scheduling_algorithm = scheduling_algorithm;
+    }
+}
+
+void Run2::set_measure_latency()
+{
+    for (auto &params: m_network_params) {
+        params.measure_hw_latency = m_measure_hw_latency;
+        params.measure_overall_latency = m_measure_overall_latency;
+    }
+}
+
+bool Run2::get_multi_process_service()
+{
+    return m_multi_process_service;
+}
+
+const std::string &Run2::get_group_id()
+{
+    return m_group_id;
+}
+
+
 /** Run2Command */
 Run2Command::Run2Command(CLI::App &parent_app) : Command(parent_app.add_subcommand(std::make_shared<Run2>()))
 {
@@ -304,10 +430,19 @@ static hailo_status wait_for_threads(std::vector<AsyncThreadPtr<hailo_status>> &
     return last_error_status;
 }
 
+bool is_valid_ip(const std::string &ip)
+{
+    int a,b,c,d;
+    return (4 == sscanf(ip.c_str(),"%d.%d.%d.%d", &a, &b, &c, &d)) &&
+        IS_FIT_IN_UINT8(a) && IS_FIT_IN_UINT8(b) && IS_FIT_IN_UINT8(c) && IS_FIT_IN_UINT8(d);
+}
+
 hailo_status Run2Command::execute()
 {
     Run2 *app = reinterpret_cast<Run2*>(m_app);
 
+    app->set_measure_latency();
+
     if (0 == app->get_network_params().size()) {
         LOGGER__ERROR("Nothing to run");
         return HAILO_INVALID_OPERATION;
@@ -316,9 +451,26 @@ hailo_status Run2Command::execute()
         LOGGER__WARN("\"hailortcli run2\" is in preview. It is recommended to use \"hailortcli run\" command for a single network group");
     }
 
-    // TODO: support multi-device. maybe get all by default?
     hailo_vdevice_params_t vdevice_params = {};
     CHECK_SUCCESS(hailo_init_vdevice_params(&vdevice_params));
+    auto dev_ids = app->get_dev_ids();
+    if (!dev_ids.empty()) {
+        vdevice_params.device_count = static_cast<uint32_t>(dev_ids.size());
+        vdevice_params.device_ids = dev_ids.data();
+
+        // Disable scheduler for eth VDevice
+        if ((1 == dev_ids.size()) && (is_valid_ip(dev_ids[0].id))) {
+            vdevice_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_NONE;
+            CHECK(1 == app->get_network_params().size(), HAILO_INVALID_OPERATION, "On Ethernet inference only one model is allowed");
+            app->set_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_NONE);
+        }
+    } else {
+        vdevice_params.device_count = app->get_device_count();
+    }
+
+    vdevice_params.group_id = app->get_group_id().c_str();
+    vdevice_params.multi_process_service = app->get_multi_process_service();
+
     auto vdevice = VDevice::create(vdevice_params);
     CHECK_EXPECTED_AS_STATUS(vdevice);
 
@@ -327,22 +479,43 @@ hailo_status Run2Command::execute()
     for (auto &net_params : app->get_network_params()) {
         auto net_runner = NetworkRunner::create_shared(*vdevice->get(), net_params);
         CHECK_EXPECTED_AS_STATUS(net_runner);
+
         net_runners.emplace_back(net_runner.release());
     }
-    LivePrinter live_printer(std::chrono::seconds(1));
+    auto live_printer = std::make_unique<LivePrinter>(std::chrono::seconds(1));
-    live_printer.add(std::make_shared<TimerLiveTrack>(app->get_time_to_run()));
+
+    live_printer->add(std::make_shared<TimerLiveTrack>(app->get_time_to_run()), 0);
 
     auto shutdown_event = Event::create(Event::State::not_signalled);
     CHECK_EXPECTED_AS_STATUS(shutdown_event);
     std::vector<AsyncThreadPtr<hailo_status>> threads;
+    Barrier barrier(net_runners.size() + 1); // We wait for all nets to finish activation + this thread to start sampling
     for (auto &net_runner : net_runners) {
-        threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>([&net_runner, &shutdown_event, &live_printer](){
+        threads.emplace_back(std::make_unique<AsyncThread<hailo_status>>("NG_INFER", [&net_runner, &shutdown_event,
-            return net_runner->run(shutdown_event.value(), live_printer);
+            &live_printer, &barrier](){
+            return net_runner->run(shutdown_event.value(), *live_printer, barrier);
         }));
     }
+
+    auto physical_devices = vdevice.value()->get_physical_devices();
+    CHECK_EXPECTED_AS_STATUS(physical_devices);
+
+    for (auto &device : physical_devices.value()) {
+        auto measurement_live_track = MeasurementLiveTrack::create_shared(device.get(), app->get_measure_power(),
+            app->get_measure_current(), app->get_measure_temp());
+        CHECK_EXPECTED_AS_STATUS(measurement_live_track);
+        live_printer->add(measurement_live_track.release(), 2);
+    }
+
     // TODO: wait for all nets before starting timer. start() should update TimerLiveTrack to start. or maybe append here but first in vector...
-    live_printer.start();
+    barrier.arrive_and_wait();
-    std::this_thread::sleep_for(app->get_time_to_run());
+    CHECK_SUCCESS(live_printer->start());
+    auto status = shutdown_event->wait(app->get_time_to_run());
+    if (HAILO_TIMEOUT != status) {
+        // if shutdown_event is signaled its because one of the send/recv threads failed
+        LOGGER__ERROR("Encountered error during inference. See log for more information.");
+    }
+    live_printer.reset(); // Ensures that the final print will include real values and not with values of when streams are already aborted.
     shutdown_event->signal();
     return wait_for_threads(threads);
 }

hailort/hailortcli/run2/timer_live_track.cpp

@@ -13,14 +13,25 @@
 #include <sstream>
 
 TimerLiveTrack::TimerLiveTrack(std::chrono::milliseconds duration) :
-        m_duration(duration), m_start(std::chrono::steady_clock::now())
+        LivePrinter::Track(), m_duration(duration), m_start_time()
 {
 }
 
+hailo_status TimerLiveTrack::start()
+{
+    m_start_time = std::chrono::steady_clock::now();
+    m_started = true;
+
+    return HAILO_SUCCESS;
+}
+
 uint32_t TimerLiveTrack::get_text(std::stringstream &ss)
 {
+    if (!m_started) {
+        return 0;
+    }
     static const uint32_t MAX_PROGRESS_BAR_WIDTH = 20;
-    auto elapsed_time = std::chrono::steady_clock::now() - m_start;
+    auto elapsed_time = std::chrono::steady_clock::now() - m_start_time;
     auto eta = std::chrono::seconds(std::max<int32_t>(0, static_cast<int32_t>(std::round(std::chrono::duration<double>(m_duration - elapsed_time).count())))); // std::chrono::round is from C++17
     auto elapsed_percentage = std::min<uint32_t>(100, static_cast<uint32_t>(std::round(std::chrono::duration<double>(100 * elapsed_time / m_duration).count())));
     auto progress_bar_width = std::max<uint32_t>(1, std::min<uint32_t>(MAX_PROGRESS_BAR_WIDTH,

hailort/hailortcli/run2/timer_live_track.hpp

@@ -17,11 +17,12 @@ class TimerLiveTrack : public LivePrinter::Track
 public:
     TimerLiveTrack(std::chrono::milliseconds duration);
     virtual ~TimerLiveTrack() = default;
+    virtual hailo_status start() override;
     virtual uint32_t get_text(std::stringstream &ss) override;
 
 private:
     std::chrono::milliseconds m_duration;
-    std::chrono::time_point<std::chrono::steady_clock> m_start;
+    std::chrono::time_point<std::chrono::steady_clock> m_start_time;
 };
 
 #endif /* _HAILO_HAILORTCLI_RUN2_TIMER_LIVE_TRACK_HPP_ */

hailort/hailortcli/run_command.cpp

@@ -11,7 +11,6 @@
 #include "hailortcli.hpp"
 #include "inference_progress.hpp"
 #include "infer_stats_printer.hpp"
-#include "temp_measurement.hpp"
 #include "graph_printer.hpp"
 #if defined(__GNUC__)
 // TODO: Support on windows (HRT-5919)
@@ -25,10 +24,13 @@
 #include "common/barrier.hpp"
 #include "common/latency_meter.hpp"
 #include "common/filesystem.hpp"
+#include "common/device_measurements.hpp"
+#include "hailo/hailort.h"
 #include "hailo/network_group.hpp"
 #include "hailo/hef.hpp"
 #include "hailo/vstream.hpp"
 #include "hailo/vdevice.hpp"
+#include "hailo/transform.hpp"
 
 #include "spdlog/fmt/fmt.h"
 
@@ -43,7 +45,6 @@ std::condition_variable wait_for_exit_cv;
     They're useful for simple interprocess communication. */
 #define USER_SIGNAL (SIGUSR1)
 
-constexpr size_t OVERALL_LATENCY_TIMESTAMPS_LIST_LENGTH (512);
 constexpr uint32_t DEFAULT_TIME_TO_RUN_SECONDS = 5;
 #ifndef HAILO_EMULATOR
 constexpr std::chrono::milliseconds TIME_TO_WAIT_FOR_CONFIG(300);
@@ -226,8 +227,8 @@ static void add_run_command_params(CLI::App *run_subcommand, inference_runner_pa
         "Collect runtime data to be used by the Profiler");
     static const char *JSON_SUFFIX = ".json";
     collect_runtime_data_subcommand->add_option("--output-path", params.runtime_data.runtime_data_output_path,
-        fmt::format("Runtime data output file path\n'{}' will be replaced with the current running hef",
+        fmt::format("Runtime data output file path\n'{}' will be replaced with the current running hef", RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER)
-            RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER))
+         + "\nIn case of multiple-devices, <device-id>_ will be added as prefix to each file")
         ->default_val(fmt::format("runtime_data_{}.json", RUNTIME_DATA_OUTPUT_PATH_HEF_PLACE_HOLDER))
         ->check(FileSuffixValidator(JSON_SUFFIX));
     collect_runtime_data_subcommand->add_option("--batch-to-measure", params.runtime_data.batch_to_measure_str,
@@ -286,9 +287,6 @@ static void add_run_command_params(CLI::App *run_subcommand, inference_runner_pa
             params.time_to_run = DEFAULT_TIME_TO_RUN_SECONDS;
         }
 
-        PARSE_CHECK(((!params.runtime_data.collect_runtime_data) || (params.vdevice_params.device_count == 1)),
-            "Passing runtime data is not supported for multiple devices");
-
         PARSE_CHECK((!(params.runtime_data.collect_runtime_data && params.vdevice_params.multi_process_service)),
             "Passing runtime data is not supported for multi process service");
 
@@ -670,7 +668,7 @@ static hailo_status run_streaming_impl(std::shared_ptr<ConfiguredNetworkGroup> c
     auto first = true;
     for (auto& recv_object : recv_objects) {
         auto &frames_recieved = frames_recieved_per_output[output_index];
-        results.emplace_back(std::make_unique<AsyncThread<hailo_status>>(
+        results.emplace_back(std::make_unique<AsyncThread<hailo_status>>("RECV",
             [network_progress_bar, params, &recv_object, &output_buffers, first, &barrier, &overall_latency_meter,
             &frames_recieved, batch_size]() {
                 auto res = recv_loop(params, recv_object.get(), network_progress_bar, barrier, overall_latency_meter,
@@ -685,7 +683,7 @@ static hailo_status run_streaming_impl(std::shared_ptr<ConfiguredNetworkGroup> c
         ++output_index;
     }
     for (auto &send_object : send_objects) {
-        results.emplace_back(std::make_unique<AsyncThread<hailo_status>>(
+        results.emplace_back(std::make_unique<AsyncThread<hailo_status>>("SEND",
             [params, &send_object, &input_dataset, &barrier, &overall_latency_meter, batch_size]() -> hailo_status {
                 auto res = send_loop(params, send_object.get(), input_dataset, barrier, overall_latency_meter, batch_size);
                 if (HAILO_SUCCESS != res) {
@@ -804,7 +802,7 @@ static Expected<InferResult> run_streaming(const std::vector<std::shared_ptr<Con
             CHECK_AS_EXPECTED(contains(recv_objects_per_network_group[network_group_index], network_name_pair.first), HAILO_INTERNAL_FAILURE,
                 "Not all networks was parsed correctly.");
             auto network_name = network_name_pair.first;
-            networks_threads_status[network_group_index].emplace_back(std::make_unique<AsyncThread<hailo_status>>(
+            networks_threads_status[network_group_index].emplace_back(std::make_unique<AsyncThread<hailo_status>>(fmt::format("NG_INFER {}", network_group_index),
                 [network_group_index, &configured_net_groups, &input_datasets, &output_buffers, &params, &send_objects_per_network_group,
                     &recv_objects_per_network_group, network_name, &progress_bar, &networks_results]() {
                     return run_streaming_impl(configured_net_groups[network_group_index], input_datasets[network_group_index],
@@ -1108,9 +1106,10 @@ Expected<InferResult> activate_and_run_single_device(
     }
 
     bool should_measure_temp = params.measure_temp;
-    TemperatureMeasurement temp_measure(device);
+    auto temp_measure = TemperatureMeasurement::create_shared(device);
+    CHECK_EXPECTED(temp_measure);
     if (should_measure_temp) {
-        auto status = temp_measure.start_measurement();
+        auto status = temp_measure.value()->start_measurement();
         CHECK_SUCCESS_AS_EXPECTED(status, "Failed to get chip's temperature");
     }
 
@@ -1136,8 +1135,8 @@ Expected<InferResult> activate_and_run_single_device(
     }
 
     if (should_measure_temp) {
-        temp_measure.stop_measurement();
+        temp_measure.value()->stop_measurement();
-        auto temp_measure_p = make_shared_nothrow<TempMeasurementData>(temp_measure.get_data());
+        auto temp_measure_p = make_shared_nothrow<AccumulatorResults>(temp_measure.value()->get_data());
         CHECK_NOT_NULL_AS_EXPECTED(temp_measure_p, HAILO_OUT_OF_HOST_MEMORY);
         auto status = inference_result.set_temp_measurement(device.get_dev_id(), std::move(temp_measure_p));
         CHECK_SUCCESS_AS_EXPECTED(status);
@@ -1257,11 +1256,11 @@ Expected<InferResult> activate_and_run_vdevice(
     std::map<std::string, std::shared_ptr<TemperatureMeasurement>> temp_measurements;
     if (params.measure_temp) {
         for (auto &device : physical_devices) {
-            auto temp_measure = make_shared_nothrow<TemperatureMeasurement>(device);
+            auto temp_measure = TemperatureMeasurement::create_shared(device);
-            CHECK_NOT_NULL_AS_EXPECTED(temp_measure, HAILO_OUT_OF_HOST_MEMORY);
+            CHECK_EXPECTED(temp_measure);
-            auto status = temp_measure->start_measurement();
+            auto status = temp_measure.value()->start_measurement();
             CHECK_SUCCESS_AS_EXPECTED(status, "Failed starting temperature measurement on device {}", device.get().get_dev_id());
-            temp_measurements.emplace(device.get().get_dev_id(), std::move(temp_measure));
+            temp_measurements.emplace(device.get().get_dev_id(), temp_measure.release());
         }
     }
 
@@ -1298,7 +1297,7 @@ Expected<InferResult> activate_and_run_vdevice(
     if (params.measure_temp) {
         for(const auto &temp_measure_pair : temp_measurements) {
             temp_measure_pair.second->stop_measurement();
-            auto temp_measure_p = make_shared_nothrow<TempMeasurementData>(temp_measure_pair.second->get_data());
+            auto temp_measure_p = make_shared_nothrow<AccumulatorResults>(temp_measure_pair.second->get_data());
             CHECK_NOT_NULL_AS_EXPECTED(temp_measure_p, HAILO_OUT_OF_HOST_MEMORY);
             auto status = inference_result.set_temp_measurement(temp_measure_pair.first, std::move(temp_measure_p));
             CHECK_SUCCESS_AS_EXPECTED(status);
@@ -1384,8 +1383,9 @@ Expected<InferResult> run_command_hef_vdevice(const inference_runner_params &par
         }
 
         if (params.runtime_data.collect_runtime_data) {
-            const auto runtime_data_output_path = format_runtime_data_output_path(
+            auto output_path = (1 == physical_devices.size()) ? params.runtime_data.runtime_data_output_path :
-                params.runtime_data.runtime_data_output_path, params.hef_path);
+                (std::string(device.get().get_dev_id()) + "_" + params.runtime_data.runtime_data_output_path);
+            const auto runtime_data_output_path = format_runtime_data_output_path(output_path, params.hef_path);
             DownloadActionListCommand::execute(device.get(), runtime_data_output_path, network_group_list.value(),
                 params.hef_path);
         }

hailort/hailortcli/run_command.hpp

@@ -13,7 +13,7 @@
 #include "hailortcli.hpp"
 #include "common.hpp"
 #include "power_measurement_command.hpp"
-#include "temp_measurement.hpp"
+#include "common/device_measurements.hpp"
 #include "CLI/CLI.hpp"
 #include "inference_result.hpp"
 

hailort/hailortcli/temp_measurement.cpp

@@ -1,83 +0,0 @@
-/**
- * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
- * Distributed under the MIT license (https://opensource.org/licenses/MIT)
- **/
-/**
- * @file temp_measurement.cpp
- * @brief Measure temperature of Hailo chip
- **/
-
-#include "temp_measurement.hpp"
-
-constexpr std::chrono::milliseconds DEFAULT_TEMPERATURE_MEASUREMENTS_INTERVAL(1000);
-
-static float32_t calc_avg(uint32_t old_samples_count, float32_t old_avg, uint32_t new_samples_count, float32_t new_value)
-{
-    float32_t old_samples = static_cast<float32_t>(old_samples_count);
-    float32_t new_samples = static_cast<float32_t>(new_samples_count);
-    float32_t total_samples_count = old_samples + new_samples;
-    return (((old_avg * old_samples) + (new_value * new_samples)) / total_samples_count);
-}
-
-TemperatureMeasurement::TemperatureMeasurement(Device &device) :
-    m_device(device),
-    m_is_thread_running(false),
-    m_data()
-{}
-
-TemperatureMeasurement::~TemperatureMeasurement()
-{
-    stop_measurement();
-}
-
-hailo_status TemperatureMeasurement::start_measurement()
-{
-    // Checking temperature sensor before starting thread
-    auto temp_info = m_device.get_chip_temperature();
-    CHECK_EXPECTED_AS_STATUS(temp_info);
-
-    m_is_thread_running = true;
-    m_thread = std::thread([this] () {
-        while (m_is_thread_running.load()) {
-            auto temp_info = m_device.get_chip_temperature();
-            if (temp_info.status() != HAILO_SUCCESS) {
-                LOGGER__ERROR("Failed to get chip's temperature, status = {}", temp_info.status());
-                m_is_thread_running = false;
-                break;
-            }
-
-            TempMeasurementData new_data = {};
-            auto old_data = m_data;
-
-            float32_t ts_avg = ((temp_info->ts0_temperature + temp_info->ts1_temperature) / 2);
-            new_data.max_value  = std::max(old_data.max_value, ts_avg);
-            new_data.min_value  = (old_data.min_value == 0) ? ts_avg : std::min(old_data.min_value, ts_avg);
-            new_data.average_value = calc_avg(old_data.sample_count, old_data.average_value, temp_info->sample_count, ts_avg);
-            new_data.sample_count = old_data.sample_count + temp_info->sample_count;
-
-            {
-                std::unique_lock<std::mutex> lock(m_mutex);
-                m_data = new_data;
-            }
-            
-            std::this_thread::sleep_for(DEFAULT_TEMPERATURE_MEASUREMENTS_INTERVAL); 
-        }
-    });
-
-    return HAILO_SUCCESS;
-}
-
-void TemperatureMeasurement::stop_measurement()
-{
-    m_is_thread_running = false;
-
-    if (m_thread.joinable()) {
-        m_thread.join();
-    }
-}
-
-const TempMeasurementData TemperatureMeasurement::get_data()
-{
-    std::unique_lock<std::mutex> lock(m_mutex);
-    return m_data;
-}

hailort/hailortcli/temp_measurement.hpp

@@ -1,48 +0,0 @@
-/**
- * Copyright (c) 2020-2022 Hailo Technologies Ltd. All rights reserved.
- * Distributed under the MIT license (https://opensource.org/licenses/MIT)
- **/
-/**
- * @file temp_measurement.hpp
- * @brief Measure temperature of Hailo chip
- **/
-
-#ifndef _HAILO_TEMP_MEASUREMENT_HPP_
-#define _HAILO_TEMP_MEASUREMENT_HPP_
-
-#include "hailortcli.hpp"
-#include "command.hpp"
-#include "hailo/hailort.h"
-#include "hailo/device.hpp"
-#include "CLI/CLI.hpp"
-
-#include <thread>
-
-struct TempMeasurementData {
-    float32_t average_value;
-    float32_t min_value;
-    float32_t max_value;
-    uint32_t sample_count;
-};
-
-
-class TemperatureMeasurement final {
-public:
-    TemperatureMeasurement(Device &device);
-    virtual ~TemperatureMeasurement();
-
-    hailo_status start_measurement();
-    void stop_measurement();
-    const TempMeasurementData get_data();
-
-private:
-    void measure_temp();
-
-    Device &m_device;
-    std::thread m_thread;
-    std::atomic_bool m_is_thread_running;
-    std::mutex m_mutex;
-    TempMeasurementData m_data;
-};
-
-#endif /* _HAILO_TEMP_MEASUREMENT_HPP_ */

hailort/libhailort/CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.0.0)
 # set(CMAKE_C_CLANG_TIDY "clang-tidy;-checks=*")
 
 set(HAILORT_MAJOR_VERSION    4)
-set(HAILORT_MINOR_VERSION    12)
+set(HAILORT_MINOR_VERSION    13)
-set(HAILORT_REVISION_VERSION 1)
+set(HAILORT_REVISION_VERSION 0)
 
 # Add the cmake folder so the modules there are found
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
@@ -48,6 +48,7 @@ add_library(readerwriterqueue INTERFACE)
 target_include_directories(readerwriterqueue INTERFACE ${HAILO_EXTERNAL_DIR}/readerwriterqueue)
 
 add_subdirectory(src)
+set(NET_FLOW_INFRA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/infra/net_flow")
 
 if(HAILO_BUILD_EXAMPLES)
     add_subdirectory(examples)

hailort/libhailort/bindings/gstreamer/CMakeLists.txt

@@ -8,7 +8,7 @@ if(NOT CMAKE_HOST_UNIX)
     message(FATAL_ERROR "Only unix hosts are supported, stopping build")
 endif()
 
-find_package(HailoRT 4.12.1 EXACT REQUIRED)
+find_package(HailoRT 4.13.0 EXACT REQUIRED)
 
 # GST_PLUGIN_DEFINE needs PACKAGE to be defined
 set(GST_HAILO_PACKAGE_NAME "hailo")
@@ -32,6 +32,8 @@ add_library(gsthailo SHARED
     gst-hailo/metadata/tensor_meta.cpp
     gst-hailo/hailo_events/hailo_events.cpp)
 
+set_property(TARGET gsthailo PROPERTY CXX_STANDARD 14)
+
 set_target_properties(gsthailo PROPERTIES
     PUBLIC_HEADER "gst-hailo/metadata/tensor_meta.hpp"
 )

hailort/libhailort/bindings/gstreamer/gst-hailo/common.hpp

@@ -48,7 +48,7 @@ using namespace hailort;
 #define DEFAULT_VDEVICE_KEY (0)
 #define MIN_VALID_VDEVICE_KEY (1)
 
-#define HAILO_SUPPORTED_FORMATS "{ RGB, RGBA, YUY2, NV12, NV21 }"
+#define HAILO_SUPPORTED_FORMATS "{ RGB, RGBA, YUY2, NV12, NV21, I420 }"
 #define HAILO_VIDEO_CAPS GST_VIDEO_CAPS_MAKE(HAILO_SUPPORTED_FORMATS)
 
 #define HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS (0)

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailodevicestats.hpp

@@ -25,6 +25,7 @@
 
 #include <memory>
 #include <mutex>
+#include <thread>
 
 G_BEGIN_DECLS
 

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.cpp

@@ -52,6 +52,29 @@ gst_scheduling_algorithm_get_type (void)
     return scheduling_algorithm_type;
 }
 
+#define GST_TYPE_HAILO_FORMAT_TYPE (gst_hailo_format_type_get_type ())
+static GType
+gst_hailo_format_type_get_type (void)
+{
+    static GType format_type_enum = 0;
+
+    /* Tightly coupled to hailo_format_type_t */
+    
+    if (!format_type_enum) {
+        static GEnumValue format_types[] = {
+            { HAILO_FORMAT_TYPE_AUTO,     "auto",     "HAILO_FORMAT_TYPE_AUTO"},
+            { HAILO_FORMAT_TYPE_UINT8,    "uint8",    "HAILO_FORMAT_TYPE_UINT8"},
+            { HAILO_FORMAT_TYPE_UINT16,   "uint16",   "HAILO_FORMAT_TYPE_UINT16"},
+            { HAILO_FORMAT_TYPE_FLOAT32,  "float32",  "HAILO_FORMAT_TYPE_FLOAT32"},
+            { HAILO_FORMAT_TYPE_MAX_ENUM,  NULL,      NULL },
+        };
+
+        format_type_enum = g_enum_register_static ("GstHailoFormatTypes", format_types);
+    }
+
+    return format_type_enum;
+}
+
 constexpr std::chrono::milliseconds WAIT_FOR_FLUSH_TIMEOUT_MS(1000);
 
 static void gst_hailonet_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec);
@@ -80,6 +103,10 @@ enum
     PROP_SCHEDULER_TIMEOUT_MS,
     PROP_SCHEDULER_THRESHOLD,
     PROP_MULTI_PROCESS_SERVICE,
+    PROP_INPUT_QUANTIZED,
+    PROP_OUTPUT_QUANTIZED,
+    PROP_INPUT_FORMAT_TYPE,
+    PROP_OUTPUT_FORMAT_TYPE,
 };
 
 G_DEFINE_TYPE(GstHailoNet, gst_hailonet, GST_TYPE_BIN);
@@ -150,8 +177,7 @@ static void gst_hailonet_class_init(GstHailoNetClass *klass)
         g_param_spec_enum("scheduling-algorithm", "Scheduling policy for automatic network group switching", "Controls the Model Scheduler algorithm of HailoRT. "
             "Gets values from the enum GstHailoSchedulingAlgorithms. "
             "Using Model Scheduler algorithm different than HAILO_SCHEDULING_ALGORITHM_NONE, excludes the property 'is-active'. "
-            "When using the same VDevice across multiple hailonets, all should have the same 'scheduling-algorithm'. "
+            "When using the same VDevice across multiple hailonets, all should have the same 'scheduling-algorithm'. ",
-            "To run with more than one device, set env variable 'HAILO_ENABLE_MULTI_DEVICE_SCHEDULER' to 1.",
             GST_TYPE_SCHEDULING_ALGORITHM, HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN,
         (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
     g_object_class_install_property(gobject_class, PROP_SCHEDULER_TIMEOUT_MS,
@@ -165,6 +191,24 @@ static void gst_hailonet_class_init(GstHailoNetClass *klass)
         g_param_spec_boolean("multi-process-service", "Should run over HailoRT service", "Controls wether to run HailoRT over its service. "
             "To use this property, the service should be active and scheduling-algorithm should be set. Defaults to false.",
             HAILO_DEFAULT_MULTI_PROCESS_SERVICE, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_INPUT_QUANTIZED,
+        g_param_spec_boolean("input-quantized", "Is the input quantized or not", "Passing `true` under the argument means that the input data sent to the stream is quantized to begin with."
+            "This will result in an input stream that doesn't quantize the input data. Passing `false` under the argument, will lead to input data being quantized.",
+            true, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_OUTPUT_QUANTIZED,
+        g_param_spec_boolean("output-quantized", "Should the output be quantized or de-quantized","Passing `true` under the argument means that the output data received from the stream is to remain quantized" 
+            "(such as it is upon exiting the device). This will result in an output stream that doesn't de-quantize the output data. Passing `false` under the argument will lead to output data being de-quantized.",
+            true, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_INPUT_FORMAT_TYPE,
+        g_param_spec_enum("input-format-type", "Input format type", "Input format type(auto, float32, uint16, uint8). Default value is auto."
+            "Gets values from the enum GstHailoFormatType. ",
+            GST_TYPE_HAILO_FORMAT_TYPE, HAILO_FORMAT_TYPE_AUTO,
+        (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+    g_object_class_install_property(gobject_class, PROP_OUTPUT_FORMAT_TYPE,
+        g_param_spec_enum("output-format-type", "Output format type", "Output format type(auto, float32, uint16, uint8). Default value is auto."
+            "Gets values from the enum GstHailoFormatType. ",
+            GST_TYPE_HAILO_FORMAT_TYPE, HAILO_FORMAT_TYPE_AUTO,
+        (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
     // See information about the "flush" signal in the element description
     g_signal_new(
         "flush",
@@ -457,6 +501,34 @@ void HailoNetImpl::set_property(GObject *object, guint property_id, const GValue
         }
         m_props.m_multi_process_service = g_value_get_boolean(value);
         break;
+    case PROP_INPUT_QUANTIZED:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the quantized flag will not take place!");
+            break;
+        }
+        m_props.m_input_quantized = g_value_get_boolean(value);
+        break;
+    case PROP_OUTPUT_QUANTIZED:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the quantized flag will not take place!");
+            break;
+        }
+        m_props.m_output_quantized = g_value_get_boolean(value);
+        break;
+    case PROP_INPUT_FORMAT_TYPE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the format type will not take place!");
+            break;
+        }
+        m_props.m_input_format_type = static_cast<hailo_format_type_t>(g_value_get_enum(value));
+        break;
+    case PROP_OUTPUT_FORMAT_TYPE:
+        if (m_was_configured) {
+            g_warning("The network was already configured so changing the format type will not take place!");
+            break;
+        }
+        m_props.m_output_format_type = static_cast<hailo_format_type_t>(g_value_get_enum(value));
+        break;
     default:
         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
         break;
@@ -527,6 +599,18 @@ void HailoNetImpl::get_property(GObject *object, guint property_id, GValue *valu
     case PROP_MULTI_PROCESS_SERVICE:
         g_value_set_boolean(value, m_props.m_multi_process_service.get());
         break;
+    case PROP_INPUT_QUANTIZED:
+        g_value_set_boolean(value, m_props.m_input_quantized.get());
+        break;
+    case PROP_OUTPUT_QUANTIZED:
+        g_value_set_boolean(value, m_props.m_output_quantized.get());
+        break;
+    case PROP_INPUT_FORMAT_TYPE:
+        g_value_set_enum(value, m_props.m_input_format_type.get());
+        break;
+    case PROP_OUTPUT_FORMAT_TYPE:
+        g_value_set_enum(value, m_props.m_output_format_type.get());
+        break;
     default:
         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
         break;
@@ -613,7 +697,8 @@ hailo_status HailoNetImpl::configure_network_group()
         GST_CHECK_SUCCESS(status, m_element, RESOURCE, "Setting scheduler threshold failed, status = %d", status);
     }
 
-    auto vstreams = m_net_group_handle->create_vstreams(m_props.m_network_name.get(), m_props.m_scheduling_algorithm.get(), m_output_formats);
+    auto vstreams = m_net_group_handle->create_vstreams(m_props.m_network_name.get(), m_props.m_scheduling_algorithm.get(), m_output_formats, static_cast<bool>(m_props.m_input_quantized.get()), 
+        static_cast<bool>(m_props.m_output_quantized.get()), m_props.m_input_format_type.get(), m_props.m_output_format_type.get());
     GST_CHECK_EXPECTED_AS_STATUS(vstreams, m_element, RESOURCE, "Creating vstreams failed, status = %d", status);
 
     GST_HAILOSEND(m_hailosend)->impl->set_input_vstreams(std::move(vstreams->first));

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailonet.hpp

@@ -54,7 +54,9 @@ public:
     HailoNetProperties() : m_device_id(nullptr), m_hef_path(nullptr), m_network_name(nullptr), m_batch_size(HAILO_DEFAULT_BATCH_SIZE),
         m_is_active(false), m_device_count(0), m_vdevice_key(DEFAULT_VDEVICE_KEY), m_scheduling_algorithm(HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN),
         m_scheduler_timeout_ms(HAILO_DEFAULT_SCHEDULER_TIMEOUT_MS), m_scheduler_threshold(HAILO_DEFAULT_SCHEDULER_THRESHOLD),
-        m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE)
+        m_multi_process_service(HAILO_DEFAULT_MULTI_PROCESS_SERVICE), m_input_quantized(true), m_output_quantized(true), m_input_format_type(HAILO_FORMAT_TYPE_AUTO),
+        m_output_format_type(HAILO_FORMAT_TYPE_AUTO)
+
     {}
 
     HailoElemProperty<gchar*> m_device_id;
@@ -68,6 +70,10 @@ public:
     HailoElemProperty<guint32> m_scheduler_timeout_ms;
     HailoElemProperty<guint32> m_scheduler_threshold;
     HailoElemProperty<gboolean> m_multi_process_service;
+    HailoElemProperty<gboolean> m_input_quantized;
+    HailoElemProperty<gboolean> m_output_quantized;
+    HailoElemProperty<hailo_format_type_t> m_input_format_type;
+    HailoElemProperty<hailo_format_type_t> m_output_format_type;
 };
 
 class HailoNetImpl final

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailorecv.cpp

@@ -281,7 +281,7 @@ hailo_status HailoRecvImpl::set_output_vstreams(std::vector<OutputVStream> &&out
     for (auto &out_vstream : m_output_vstreams) {
         GstHailoBufferPool *hailo_pool = GST_HAILO_BUFFER_POOL(g_object_new(GST_TYPE_HAILO_BUFFER_POOL, NULL));
         gst_object_ref_sink(hailo_pool);
-        memcpy(hailo_pool->vstream_name, out_vstream.name().c_str(), sizeof(hailo_pool->vstream_name));
+        strncpy(hailo_pool->vstream_name, out_vstream.name().c_str(), out_vstream.name().length() + 1);
         hailo_pool->element_name = GST_ELEMENT_NAME(GST_ELEMENT_PARENT(m_element));
 
         GstBufferPool *pool = GST_BUFFER_POOL(hailo_pool);

hailort/libhailort/bindings/gstreamer/gst-hailo/gsthailosend.cpp

@@ -33,6 +33,7 @@ GST_DEBUG_CATEGORY_STATIC(gst_hailosend_debug_category);
 #define YUY2_FEATURES_SIZE (2)
 #define NV12_FEATURES_SIZE (3)
 #define NV21_FEATURES_SIZE (3)
+#define I420_FEATURES_SIZE (3)
 
 static void gst_hailosend_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec);
 static void gst_hailosend_get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec);
@@ -64,7 +65,7 @@ static void gst_hailosend_class_init(GstHailoSendClass *klass)
         gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, gst_caps_from_string(HAILO_VIDEO_CAPS)));
 
     gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass),
-        "hailosend element", "Hailo/Filter/Video", "Send RGB/RGBA/YUY2 video to HailoRT", PLUGIN_AUTHOR);
+        "hailosend element", "Hailo/Filter/Video", "Send RGB/RGBA/YUY2/NV12/NV21/I420 video to HailoRT", PLUGIN_AUTHOR);
     
     element_class->change_state = GST_DEBUG_FUNCPTR(gst_hailosend_change_state);
 
@@ -238,6 +239,12 @@ GstCaps *HailoSendImpl::get_caps(GstBaseTransform */*trans*/, GstPadDirection /*
             "Features of input vstream %s is not %d for NV21 format! (features=%d)", m_input_vstream_infos[0].name, NV21_FEATURES_SIZE,
             m_input_vstream_infos[0].shape.features);
         break;
+    case HAILO_FORMAT_ORDER_I420:
+        format = "I420";
+        GST_CHECK(I420_FEATURES_SIZE == m_input_vstream_infos[0].shape.features, NULL, m_element, STREAM,
+            "Features of input vstream %s is not %d for I420 format! (features=%d)", m_input_vstream_infos[0].name, I420_FEATURES_SIZE,
+            m_input_vstream_infos[0].shape.features);
+        break;
     default:
         GST_ELEMENT_ERROR(m_element, RESOURCE, FAILED,
             ("Input VStream %s has an unsupported format order! order = %d", m_input_vstream_infos[0].name, m_input_vstream_infos[0].format.order), (NULL));

hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.cpp

@@ -17,11 +17,15 @@
  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  * Boston, MA 02110-1301, USA.
  */
+
+#include "hailo/hailort_common.hpp"
+
 #include "network_group_handle.hpp"
 
 #include <sstream>
 #include <chrono>
 
+
 std::unordered_set<std::shared_ptr<VDevice>> NetworkGroupHandle::m_vdevices;
 NetworkGroupConfigManager NetworkGroupHandle::m_net_group_config_manager;
 NetworkGroupActivationManager NetworkGroupHandle::m_net_group_activation_manager;
@@ -177,7 +181,8 @@ hailo_status NetworkGroupHandle::set_scheduler_threshold(const char *network_nam
 }
 
 Expected<std::pair<std::vector<InputVStream>, std::vector<OutputVStream>>> NetworkGroupHandle::create_vstreams(const char *network_name,
-    hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector<hailo_format_with_name_t> &output_formats)
+    hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector<hailo_format_with_name_t> &output_formats, bool input_quantized, 
+    bool output_quantized, hailo_format_type_t input_format_type, hailo_format_type_t output_format_type)
 {
     GST_CHECK(nullptr != network_name, make_unexpected(HAILO_INVALID_ARGUMENT), m_element, RESOURCE, "Got nullptr in network name!");
 
@@ -191,11 +196,10 @@ Expected<std::pair<std::vector<InputVStream>, std::vector<OutputVStream>>> Netwo
     auto expected_input_vstream_infos = hef()->get_input_vstream_infos(network_name);
     GST_CHECK_EXPECTED(expected_input_vstream_infos, m_element, RESOURCE, "Failed getting input vstream infos, status = %d",
         expected_input_vstream_infos.status());
-    auto expected_input_params_map = m_cng->make_input_vstream_params(true, HAILO_FORMAT_TYPE_AUTO, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS,
+    auto expected_input_params_map = m_cng->make_input_vstream_params(input_quantized, input_format_type, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS,
         HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, m_network_name);
     GST_CHECK_EXPECTED(expected_input_params_map, m_element, RESOURCE, "Failed making input vstream params, status = %d",
         expected_input_params_map.status());
-
     // In RGB formats, Gstreamer is padding each row to 4.
     auto &&input_params_map = expected_input_params_map.release();
     auto &&input_infos = expected_input_vstream_infos.release();
@@ -215,7 +219,7 @@ Expected<std::pair<std::vector<InputVStream>, std::vector<OutputVStream>>> Netwo
     GST_CHECK(1 == input_vstreams->size(), make_unexpected(HAILO_INVALID_OPERATION), m_element, RESOURCE,
         "hailosend element supports only HEFs with one input for now!");
 
-    auto output_params_map = m_cng->make_output_vstream_params(true, HAILO_FORMAT_TYPE_AUTO, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS,
+    auto output_params_map = m_cng->make_output_vstream_params(output_quantized, output_format_type, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS,
         HAILO_DEFAULT_VSTREAM_QUEUE_SIZE, m_network_name);
     GST_CHECK_EXPECTED(output_params_map, m_element, RESOURCE, "Failed making output vstream params, status = %d",
         output_params_map.status());
@@ -244,11 +248,7 @@ Expected<std::pair<std::vector<InputVStream>, std::vector<OutputVStream>>> Netwo
 Expected<NetworkGroupsParamsMap> NetworkGroupHandle::get_configure_params(Hef &hef, const VDevice &vdevice,
     const char *net_group_name, uint16_t batch_size)
 {
-    auto stream_interface = vdevice.get_default_streams_interface();
+    auto params = vdevice.create_configure_params(hef, net_group_name);
-    GST_CHECK_EXPECTED(stream_interface, m_element, RESOURCE,
-        "Failed default stream interface configure params, status = %d", stream_interface.status());
-
-    auto params = hef.create_configure_params(*stream_interface, net_group_name);
     GST_CHECK_EXPECTED(params, m_element, RESOURCE, "Failed creating configure params, status = %d", params.status());
     params->batch_size = batch_size;
 

hailort/libhailort/bindings/gstreamer/gst-hailo/network_group_handle.hpp

@@ -83,7 +83,8 @@ public:
         bool multi_process_service, const char *hef_path);
     hailo_status configure_network_group(const char *net_group_name, hailo_scheduling_algorithm_t scheduling_algorithm, uint16_t batch_size);
     Expected<std::pair<std::vector<InputVStream>, std::vector<OutputVStream>>> create_vstreams(const char *network_name,
-        hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector<hailo_format_with_name_t> &output_formats);
+        hailo_scheduling_algorithm_t scheduling_algorithm, const std::vector<hailo_format_with_name_t> &output_formats, bool input_quantized, 
+        bool output_quantized, hailo_format_type_t input_format_type, hailo_format_type_t output_format_type);
     hailo_status activate_network_group();
     Expected<bool> remove_network_group();
 

hailort/libhailort/bindings/python/examples/hef_infer_pipeline_vstream.py

@@ -1,34 +0,0 @@
-from hailo_platform import (HEF, VDevice, ConfigureParams, InferVStreams, InputVStreamParams,
-    OutputVStreamParams, FormatType)
-from hailo_platform.pyhailort.pyhailort import HailoStreamInterface
-import numpy as np
-import argparse
-
-def parse_args():
-    parser = argparse.ArgumentParser(description='Streaming API example')
-    parser.add_argument('hef_path', type=str, help='Path of the HEF to run')
-    parser.add_argument('-n', '--num-frames', type=int, default=10, help='Number of frames to send')
-    return parser.parse_args()
-
-def main():
-    args = parse_args()
-    with VDevice() as target:
-        hef = HEF(args.hef_path)
-        configure_params = ConfigureParams.create_from_hef(hef, interface=HailoStreamInterface.PCIe)
-        network_groups = target.configure(hef, configure_params)
-        network_group = network_groups[0]
-        network_group_params = network_group.create_params()
-        input_vstreams_params = InputVStreamParams.make(network_group, quantized=False, format_type=FormatType.FLOAT32)
-        output_vstreams_params = OutputVStreamParams.make(network_group, quantized=True, format_type=FormatType.AUTO)
-        with InferVStreams(network_group, input_vstreams_params, output_vstreams_params) as infer_pipeline:
-            input_names_to_shape = {vstream_info.name: vstream_info.shape for vstream_info in hef.get_input_vstream_infos()}
-            input_data = {name : 1 + np.ndarray([args.num_frames] + list(shape), dtype=np.float32) for name, shape in input_names_to_shape.items()}
-            with network_group.activate(network_group_params):
-                _ = infer_pipeline.infer(input_data)
-                fps = args.num_frames / infer_pipeline.get_hw_time()
-
-    print('Inference ran successfully')
-    print(f'FPS: {fps}')
-
-if __name__ == '__main__':
-    main()

hailort/libhailort/bindings/python/examples/hef_infer_virtual_stream.py

@@ -1,69 +0,0 @@
-import argparse
-import time
-import numpy as np
-
-from multiprocessing import Process
-from hailo_platform import (HEF, VDevice, HailoStreamInterface, ConfigureParams, InputVStreamParams, InputVStreams,
-                            OutputVStreamParams, OutputVStreams)
-
-def send(configured_network, num_frames):
-    vstreams_params = InputVStreamParams.make(configured_network)
-    configured_network.wait_for_activation(1000)
-    with InputVStreams(configured_network, vstreams_params) as vstreams:
-        vstream_to_buffer = {vstream: np.ndarray([1] + list(vstream.shape), dtype=vstream.dtype) for vstream in vstreams}
-        for _ in range(num_frames):
-            for vstream, buff in vstream_to_buffer.items():
-                vstream.send(buff)
-        # Flushing is not mandatory here
-        for vstream in vstreams:
-            vstream.flush()
-
-def recv(configured_network, vstreams_params, num_frames):
-    configured_network.wait_for_activation(1000)
-    with OutputVStreams(configured_network, vstreams_params) as vstreams:
-        for _ in range(num_frames):
-            for vstream in vstreams:
-                _ = vstream.recv()
-
-def recv_all(configured_network, num_frames):
-    vstreams_params_groups = OutputVStreamParams.make_groups(configured_network)
-    recv_procs = []
-    for vstreams_params in vstreams_params_groups:
-        proc = Process(target=recv, args=(configured_network, vstreams_params, num_frames))
-        proc.start()
-        recv_procs.append(proc)
-
-    for proc in recv_procs:
-        proc.join()
-
-def parse_args():
-    parser = argparse.ArgumentParser(description='vStream API example')
-    parser.add_argument('hef_path', type=str, help='Path of the HEF to run')
-    parser.add_argument('-n', '--num-frames', type=int, default=1000, help='Number of frames to send')
-    return parser.parse_args()
-
-def main():
-    args = parse_args()
-    hef = HEF(args.hef_path)
-
-    with VDevice() as device:
-        configure_params = ConfigureParams.create_from_hef(hef, interface=HailoStreamInterface.PCIe)
-        network_group = device.configure(hef, configure_params)[0]
-        network_group_params = network_group.create_params()
-        send_process = Process(target=send, args=(network_group, args.num_frames))
-        recv_process = Process(target=recv_all, args=(network_group, args.num_frames))
-
-        time_before = time.time()
-        recv_process.start()
-        send_process.start()
-        with network_group.activate(network_group_params):
-            send_process.join()
-            recv_process.join()
-
-        fps = args.num_frames / (time.time() - time_before)
-
-    print('Inference ran successfully')
-    print(f'FPS: {fps}')
-
-if __name__ == '__main__':
-    main()

hailort/libhailort/bindings/python/platform/hailo_platform/__init__.py

@@ -26,7 +26,7 @@ from hailo_platform.pyhailort.pyhailort import (HEF, ConfigureParams,
                                                 InputVStreams, OutputVStreams,
                                                 InferVStreams, HailoStreamDirection, HailoFormatFlags, HailoCpuId, Device, VDevice,
                                                 DvmTypes, PowerMeasurementTypes, SamplingPeriod, AveragingFactor, MeasurementBufferIndex,
-                                                HailoRTException, YOLOv5PostProcessingOp)
+                                                HailoRTException, YOLOv5PostProcessOp, HailoSchedulingAlgorithm)
 
 def _verify_pyhailort_lib_exists():
     python_version = "".join(str(i) for i in sys.version_info[:2])
@@ -62,4 +62,4 @@ __all__ = ['EthernetDevice', 'DvmTypes', 'PowerMeasurementTypes',
            'MipiIspImageInOrder', 'MipiIspImageOutDataType', 'join_drivers_path', 'IspLightFrequency', 'HailoPowerMode',
            'Endianness', 'HailoStreamInterface', 'InputVStreamParams', 'OutputVStreamParams',
            'InputVStreams', 'OutputVStreams', 'InferVStreams', 'HailoStreamDirection', 'HailoFormatFlags', 'HailoCpuId',
-           'Device', 'VDevice', 'HailoRTException', 'YOLOv5PostProcessingOp']
+           'Device', 'VDevice', 'HailoRTException', 'YOLOv5PostProcessOp', 'HailoSchedulingAlgorithm']

hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/control_object.py

@@ -9,7 +9,6 @@ from hailo_platform.pyhailort.pyhailort import (Control, InternalPcieDevice, Exc
 
 import hailo_platform.pyhailort._pyhailort as _pyhailort
 
-
 class ControlObjectException(Exception):
     """Raised on illegal ContolObject operation."""
     pass
@@ -38,6 +37,9 @@ class UdpHcpControl(HcpControl):
             ignore_socket_errors (bool, optional): Ignore socket error (might be usefull for debugging).
         """
         # In the C API we define the total amount of attempts, instead of the amount of retries.
+
+        # TODO: HRT-9987 - Add this deprecation warning
+        # default_logger().warning("UdpHcpControl is deprecated! Please Use Control object")
         max_number_of_attempts = retries + 1
         response_timeout_milliseconds = int(response_timeout_seconds * 1000)
         if device is None:
@@ -55,6 +57,8 @@ class PcieHcpControl(HcpControl):
 
     def __init__(self, device=None, device_info=None):
         """Initializes a new HailoPcieController object."""
+        # TODO: HRT-9987 - Add this deprecation warning
+        # default_logger().warning("PcieHcpControl is deprecated! Please Use Control object")
         if device_info is None:
             device_info = InternalPcieDevice.scan_devices()[0]
 

hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/hw_object.py

@@ -27,7 +27,8 @@ class HailoHWObjectException(Exception):
 
 
 class HailoHWObject(object):
-    """Abstract Hailo hardware device representation."""
+    # TODO: HRT-9987 - Add (deprecated) to this doc
+    """Abstract Hailo hardware device representation"""
 
     NAME = InferenceTargets.UNINITIALIZED
     IS_HARDWARE = True
@@ -43,6 +44,9 @@ class HailoHWObject(object):
         self._is_device_used = False
         self._hef_loaded = False
 
+        # TODO: HRT-9987 - Add this deprecation warning
+        # self._logger.warning("HailoHWObject is deprecated! Please use VDevice/Device object.")
+
     # TODO: HRT-6310 Remove this.
     def __eq__(self, other):
         return type(self).NAME == other
@@ -50,14 +54,16 @@ class HailoHWObject(object):
     @property
     def name(self):
         """str: The name of this target. Valid values are defined by :class:`~hailo_platform.pyhailort.hw_object.InferenceTargets`"""
-        # self._logger.warning("HailoHWObject name property is deprecated! Please use VDevice object with device_id.")
+        # TODO: HRT-9987 - Add this deprecation warning
+        # self._logger.warning("HailoHWObject name property is deprecated! Please use VDevice/Device object with device_id.")
         return type(self).NAME
 
     @property
     def is_hardware(self):
         """bool: Indicates this target runs on a physical hardware device."""
         # TODO: SDK should implement in Target
-        # self._logger.warning("HailoHWObject is_hardware property is deprecated! Please use VDevice object, or derive from it.")
+        # TODO: HRT-9987 - Add this deprecation warning
+        # self._logger.warning("HailoHWObject is_hardware property is deprecated! Please use VDevice/Device object, or derive from it.")
         return type(self).IS_HARDWARE
 
     @property
@@ -74,6 +80,7 @@ class HailoHWObject(object):
         Returns:
             list of str: Sorted list of the output layer names.
         """
+        # TODO: HRT-9987 - Add this deprecation warning
         # self._logger.warning("HailoHWObject sorted_output_layer_names property is deprecated! Please use ConfiguredNetwork get_sorted_output_names.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to sorted_output_layer_names is only allowed when there is a single loaded network group")
@@ -81,8 +88,9 @@ class HailoHWObject(object):
 
     @contextmanager
     def use_device(self, *args, **kwargs):
-        # self._logger.warning("HailoHWObject use_device context manager is deprecated! Please use VDevice object.")
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
-        """A context manager that wraps the usage of the device (deprecated)."""
+        # self._logger.warning("HailoHWObject use_device context manager is deprecated! Please use VDevice/Device object.")
+        """A context manager that wraps the usage of the device."""
         self._is_device_used = True
         yield
         self._is_device_used = False
@@ -93,6 +101,7 @@ class HailoHWObject(object):
         Returns:
             dict: Keys are device output names and values are lists of layers' names.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject get_output_device_layer_to_original_layer_map function is deprecated!")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to layer names is only allowed when there is a single loaded network group")
@@ -105,6 +114,7 @@ class HailoHWObject(object):
         Returns:
             dict: Keys are the names of the layers and values are device outputs names.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject get_original_layer_to_device_layer_map function is deprecated!")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to layer names is only allowed when there is a single loaded network group")
@@ -114,30 +124,35 @@ class HailoHWObject(object):
     @property
     def device_input_layers(self):
         """Get a list of the names of the device's inputs."""
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject device_input_layers function is deprecated! Please use ConfiguredNetwork object.")
         return [layer.name for layer in self.get_input_stream_infos()]
 
     @property
     def device_output_layers(self):
         """Get a list of the names of the device's outputs."""
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject device_output_layers function is deprecated! Please use ConfiguredNetwork object.")
         return [layer.name for layer in self.get_output_stream_infos()]
 
     def hef_loaded(self):
         """Return True if this object has loaded the model HEF to the hardware device."""
         # TODO: SDK should implement in Target
-        # self._logger.warning("HailoHWObject hef_loaded function is deprecated! Please use VDevice object, or derive from it.")
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
+        # self._logger.warning("HailoHWObject hef_loaded function is deprecated! Please use VDevice/Device object, or derive from it.")
         return self._hef_loaded
 
     def outputs_count(self):
         """Return the amount of output tensors that are returned from the hardware device for every
         input image.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject outputs_count function is deprecated! Please use ConfiguredNetwork object.")
         return len(self.get_output_vstream_infos())
 
     def _clear_shapes(self):
         # TODO: SDK should implement in Target
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject _clear_shapes function is deprecated! Please use ConfiguredNetwork object.")
         self._hw_consts = None
 
@@ -148,6 +163,7 @@ class HailoHWObject(object):
         Returns:
             str: Model name.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject model_name property is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) == 1:
             return self._loaded_network_groups[0].name
@@ -160,6 +176,7 @@ class HailoHWObject(object):
         Returns:
             Tuple of output shapes, sorted by the output names.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoHWObject get_output_shapes function is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Calling get_output_shapes is only allowed when there is a single loaded network group")
@@ -167,6 +184,7 @@ class HailoHWObject(object):
 
 
 class HailoChipObject(HailoHWObject):
+    # TODO: HRT-9987 - Add (deprecated) to this docs
     """Hailo hardware device representation"""
 
     def __init__(self):
@@ -195,6 +213,7 @@ class HailoChipObject(HailoHWObject):
         Returns:
             dict of :obj:'numpy.dtype': where the key is model input_layer name, and the value is dtype as the device expect to get for this input. 
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_all_input_layers_dtype function is deprecated! Please use ConfiguredNetwork object.")
         return {stream.name: HailoRTTransformUtils.get_dtype(stream.data_bytes) for stream in self.get_input_stream_infos()}
 
@@ -208,6 +227,7 @@ class HailoChipObject(HailoHWObject):
             If there is exactly one configured network group, returns a list of
             :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with all the information objects of all input vstreams
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_input_vstream_infos function is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to network vstream info is only allowed when there is a single loaded network group")
@@ -223,6 +243,7 @@ class HailoChipObject(HailoHWObject):
             If there is exactly one configured network group, returns a list of
             :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with all the information objects of all output vstreams
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_output_vstream_infos function is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to network vstream info is only allowed when there is a single loaded network group")
@@ -238,6 +259,7 @@ class HailoChipObject(HailoHWObject):
             If there is exactly one configured network group, returns a list of
             :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with all the information objects of all input and output vstreams
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_all_vstream_infos function is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to network vstream info is only allowed when there is a single loaded network group")
@@ -254,6 +276,7 @@ class HailoChipObject(HailoHWObject):
             :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with information objects
             of all input low-level streams.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_input_stream_infos function is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to network stream info is only allowed when there is a single loaded network group")
@@ -270,6 +293,7 @@ class HailoChipObject(HailoHWObject):
             :obj:`hailo_platform.pyhailort._pyhailort.VStreamInfo`: with information objects
             of all output low-level streams.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_output_stream_infos function is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to network stream info is only allowed when there is a single loaded network group")
@@ -285,6 +309,7 @@ class HailoChipObject(HailoHWObject):
             If there is exactly one configured network group, returns a list of
             :obj:`hailo_platform.pyhailort._pyhailort.StreamInfo`: with all the information objects of all input and output streams
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_all_stream_infos function is deprecated! Please use ConfiguredNetwork object.")
         if len(self._loaded_network_groups) != 1:
             raise HailoHWObjectException("Access to network stream info is only allowed when there is a single loaded network group")
@@ -328,6 +353,7 @@ class HailoChipObject(HailoHWObject):
         Returns:
             Tuple of integers representing the input_shape.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_input_shape function is deprecated! Please use ConfiguredNetwork object.")
         if name is None:
             name = self.get_input_vstream_infos()[0].name
@@ -348,6 +374,7 @@ class HailoChipObject(HailoHWObject):
         Returns:
             int: The index of the layer name in the output list.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # self._logger.warning("HailoChipObject get_index_from_name function is deprecated! Please use ConfiguredNetwork object.")
         try:
             return self.sorted_output_layer_names.index(name)
@@ -371,7 +398,8 @@ class HailoChipObject(HailoHWObject):
 
 
 class EthernetDevice(HailoChipObject):
-    """Represents any Hailo hardware device that supports UDP control and dataflow."""
+    # TODO: HRT-9987 - Add (deprecated) to this docs
+    """Represents any Hailo hardware device that supports UDP control and dataflow"""
 
     NAME = InferenceTargets.UDP_CONTROLLER
 
@@ -414,6 +442,7 @@ class EthernetDevice(HailoChipObject):
         Returns:
             list of str: IPs of scanned devices.
         """
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
         # default_logger().warning("EthernetDevice scan_devices method is deprecated! Please use scan() of Device object.")
         udp_scanner = HailoUdpScan()
         return udp_scanner.scan_devices(interface_name, timeout_seconds=timeout_seconds)
@@ -435,12 +464,14 @@ class EthernetDevice(HailoChipObject):
     @property
     def remote_ip(self):
         """Return the IP of the remote device."""
-        # self._logger.warning("EthernetDevice remote_ip method is deprecated! Please use Device object.")
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
+        # self._logger.warning("EthernetDevice remote_ip method is deprecated! Please use VDevice/Device object.")
         return self._remote_ip
 
 
 class PcieDevice(HailoChipObject):
-    """Hailo PCIe production device representation."""
+    # TODO: HRT-9987 - Add (deprecated) to this docs
+    """Hailo PCIe production device representation"""
 
     NAME = InferenceTargets.PCIE_CONTROLLER
 
@@ -455,7 +486,8 @@ class PcieDevice(HailoChipObject):
                 :func:`PcieDevice.scan_devices` to get list of all available devices.
         """
         super(PcieDevice, self).__init__()
-        # self._logger.warning("PcieDevice deprecated! Please use VDevice object.")
+        # TODO: HRT-9987 - Add this deprecation warning
+        # self._logger.warning("PcieDevice is deprecated! Please use VDevice/Device object.")
 
         gc.collect()
         # PcieDevice __del__ function tries to release self._device.
@@ -479,7 +511,8 @@ class PcieDevice(HailoChipObject):
         Returns:
             list of :obj:`hailo_platform.pyhailort.pyhailort.PcieDeviceInfo`
         """
-        # default_logger().warning("PcieDevice scan_devices method is deprecated! Please use scan() of Device object.")
+        # TODO: HRT-9987 - Add this deprecation warning and (deprecated) to this docs
+        # default_logger().warning("PcieDevice scan_devices method is deprecated! Please use Device object.")
         return InternalPcieDevice.scan_devices()
 
     def _open_device(self, device_info):

hailort/libhailort/bindings/python/platform/hailo_platform/pyhailort/pyhailort.py

@@ -48,6 +48,8 @@ class HailoSocket(object):
     MIN_UDP_PADDED_PAYLOAD_SIZE = HailoSocketDefs.MIN_UDP_PADDED_PAYLOAD_SIZE()
     MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP = HailoSocketDefs.MAX_ALIGNED_UDP_PAYLOAD_SIZE_RTP()
 
+class HailoSchedulingAlgorithm(_pyhailort.SchedulingAlgorithm):
+    pass
 
 class HailoRTException(Exception):
     pass
@@ -104,46 +106,47 @@ class ExceptionWrapper(object):
     def __exit__(self, exception_type, value, traceback):
         if value is not None:
             if exception_type is _pyhailort.HailoRTStatusException:
-                self._raise_indicative_status_exception(int(value.args[0]))
+                self._raise_indicative_status_exception(value)
             else:
                 raise
 
-    def _raise_indicative_status_exception(self, error_code):
+    def _raise_indicative_status_exception(self, libhailort_exception):
+        error_code = int(libhailort_exception.args[0])
         string_error_code = get_status_message(error_code)
         if string_error_code == "HAILO_ETH_RECV_FAILURE":
-            raise UdpRecvError("Failed to receive data")
+            raise UdpRecvError("Failed to receive data") from libhailort_exception
         if string_error_code == "HAILO_UNSUPPORTED_CONTROL_PROTOCOL_VERSION":
-            raise InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device")
+            raise InvalidProtocolVersionException("HailoRT has failed because an invalid protocol version was received from device") from libhailort_exception
         if string_error_code == "HAILO_FW_CONTROL_FAILURE":
-            raise HailoRTFirmwareControlFailedException("libhailort control operation failed")
+            raise HailoRTFirmwareControlFailedException("libhailort control operation failed") from libhailort_exception
         if string_error_code == "HAILO_UNSUPPORTED_OPCODE":
-            raise HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device")
+            raise HailoRTUnsupportedOpcodeException("HailoRT has failed because an unsupported opcode was sent to device") from libhailort_exception
         if string_error_code == "HAILO_INVALID_FRAME":
-            raise HailoRTInvalidFrameException("An invalid frame was received")
+            raise HailoRTInvalidFrameException("An invalid frame was received") from libhailort_exception
         if string_error_code == "HAILO_TIMEOUT":
-            raise HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred")
+            raise HailoRTTimeout("Received a timeout - hailort has failed because a timeout had occurred") from libhailort_exception
         if string_error_code == "HAILO_STREAM_ABORTED_BY_HW":
-            raise HailoRTStreamAborted("Stream aborted due to an external event")
+            raise HailoRTStreamAborted("Stream aborted due to an external event") from libhailort_exception
 
         if string_error_code == "HAILO_INVALID_OPERATION":
-            raise HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information")
+            raise HailoRTInvalidOperationException("Invalid operation. See hailort.log for more information") from libhailort_exception
         if string_error_code == "HAILO_INVALID_ARGUMENT":
-            raise HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information")
+            raise HailoRTInvalidArgumentException("Invalid argument. See hailort.log for more information") from libhailort_exception
         if string_error_code == "HAILO_NOT_FOUND":
-            raise HailoRTNotFoundException("Item not found. See hailort.log for more information")
+            raise HailoRTNotFoundException("Item not found. See hailort.log for more information") from libhailort_exception
 
         if string_error_code == "HAILO_INVALID_HEF":
-            raise HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information")
+            raise HailoRTInvalidHEFException("Invalid HEF. See hailort.log for more information") from libhailort_exception
 
         if string_error_code == "HAILO_ETH_FAILURE":
-            raise HailoRTEthException("Ethernet failure. See hailort.log for more information")
+            raise HailoRTEthException("Ethernet failure. See hailort.log for more information") from libhailort_exception
-        if string_error_code == "HAILO_PCIE_DRIVER_FAIL":
+        if string_error_code == "HAILO_DRIVER_FAIL":
-            raise HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information")
+            raise HailoRTPCIeDriverException("PCIe driver failure. run 'dmesg | grep hailo' for more information") from libhailort_exception
 
         if string_error_code == "HAILO_NETWORK_GROUP_NOT_ACTIVATED":
-            raise HailoRTNetworkGroupNotActivatedException("Network group is not activated")
+            raise HailoRTNetworkGroupNotActivatedException("Network group is not activated") from libhailort_exception
         else:
-            raise HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code))
+            raise HailoRTException("libhailort failed with error: {} ({})".format(error_code, string_error_code)) from libhailort_exception
 
 def get_status_message(status_code):
     status_str = _pyhailort.get_status_message(status_code)
@@ -523,6 +526,8 @@ class ConfiguredNetwork(object):
 
     def __init__(self, configured_network, target, hef):
         self._configured_network = configured_network
+        self._input_vstreams_holders = []
+        self._output_vstreams_holders = []
         self._target = target
         self._hef = hef
 
@@ -540,6 +545,7 @@ class ConfiguredNetwork(object):
             :class:`ActivatedNetworkContextManager`: Context manager that returns the activated
             network group.
         """
+        # TODO: HRT-9988 - Add deprecation warning when changing to service by default
         network_group_params = network_group_params or self.create_params()
 
         with ExceptionWrapper():
@@ -670,11 +676,39 @@ class ConfiguredNetwork(object):
         with ExceptionWrapper():
             return self._configured_network.get_udp_rates_dict(int(fps), int(max_supported_rate_bytes))
 
+    def _before_fork(self):
+        if self._configured_network is not None:
+            self._configured_network.before_fork()
+            for input_vstreams in self._input_vstreams_holders:
+                input_vstreams.before_fork()
+            for output_vstreams in self._output_vstreams_holders:
+                output_vstreams.before_fork()
+
+    def _after_fork_in_parent(self):
+        if self._configured_network is not None:
+            self._configured_network.after_fork_in_parent()
+            for input_vstreams in self._input_vstreams_holders:
+                input_vstreams.after_fork_in_parent()
+            for output_vstreams in self._output_vstreams_holders:
+                output_vstreams.after_fork_in_parent()
+
+    def _after_fork_in_child(self):
+        if self._configured_network is not None:
+            self._configured_network.after_fork_in_child()
+            for input_vstreams in self._input_vstreams_holders:
+                input_vstreams.after_fork_in_child()
+            for output_vstreams in self._output_vstreams_holders:
+                output_vstreams.after_fork_in_child()
+
     def _create_input_vstreams(self, input_vstreams_params):
-        return self._configured_network.InputVStreams(input_vstreams_params)
+        input_vstreams_holder = self._configured_network.InputVStreams(input_vstreams_params)
+        self._input_vstreams_holders.append(input_vstreams_holder)
+        return input_vstreams_holder
 
     def _create_output_vstreams(self, output_vstreams_params):
-        return self._configured_network.OutputVStreams(output_vstreams_params)
+        output_vstreams_holder = self._configured_network.OutputVStreams(output_vstreams_params)
+        self._output_vstreams_holders.append(output_vstreams_holder)
+        return output_vstreams_holder
 
     def get_stream_names_from_vstream_name(self, vstream_name):
         """Get stream name from vstream name for a specific network group.
@@ -700,6 +734,38 @@ class ConfiguredNetwork(object):
         with ExceptionWrapper():
             return self._hef.get_vstream_names_from_stream_name(stream_name, self.name)
 
+    def set_scheduler_timeout(self, timeout_ms, network_name=None):
+        """Sets the maximum time period that may pass before getting run time from the scheduler,
+            even without reaching the minimum required send requests (e.g. threshold - see set_scheduler_threshold()),
+            as long as at least one send request has been sent.
+            This time period is measured since the last time the scheduler gave this network group run time.
+        
+        Args:
+            timeout_ms (int): Timeout in milliseconds.
+        """
+        name = network_name if network_name is not None else self.name
+        return self._configured_network.set_scheduler_timeout(timeout_ms, name)
+
+    def set_scheduler_threshold(self, threshold):
+        """Sets the minimum number of send requests required before the network is considered ready to get run time from the scheduler.
+            If at least one send request has been sent, but the threshold is not reached within a set time period (e.g. timeout - see hailo_set_scheduler_timeout()),
+            the scheduler will consider the network ready regardless.
+
+        Args:
+            threshold (int): Threshold in number of frames.
+        """
+        return self._configured_network.set_scheduler_threshold(threshold)
+
+    def set_scheduler_priority(self, priority):
+        """Sets the priority of the network.
+            When the model scheduler will choose the next network, networks with higher priority will be prioritized in the selection.
+            bigger number represent higher priority.
+
+        Args:
+            priority (int): Priority as a number between HAILO_SCHEDULER_PRIORITY_MIN - HAILO_SCHEDULER_PRIORITY_MAX.
+        """
+        return self._configured_network.set_scheduler_priority(priority)
+
 
 class ActivatedNetworkContextManager(object):
     """A context manager that returns the activated network group upon enter."""
@@ -1075,7 +1141,8 @@ class HailoRTTransformUtils(object):
 
 class InternalEthernetDevice(object):
     def __init__(self, address, port, response_timeout_seconds=10, max_number_of_attempts=3):
-        # default_logger().warning("InternalEthernetDevice is deprecated! Please use Device object.")
+        # TODO: HRT-9987 - Add this deprecation warning
+        # default_logger().warning("InternalEthernetDevice is deprecated! Please use VDevice object.")
         self.device = None
         self._address = address
         self._port = port
@@ -1104,7 +1171,6 @@ class PcieDeviceInfo(_pyhailort.PcieDeviceInfo):
 
     def __init__(self, bus, device, func, domain=None):
         super(PcieDeviceInfo, self).__init__()
-        # default_logger().warning("PcieDeviceInfo is deprecated! Please use Device object with device_id.")
         self.bus = bus
         self.device = device
         self.func = func
@@ -1141,7 +1207,6 @@ class PcieDeviceInfo(_pyhailort.PcieDeviceInfo):
 
 class InternalPcieDevice(object):
     def __init__(self, device_info=None):
-        # self._logger.warning("InternalPcieDevice deprecated! Please use Device object.")
         self.device = None
         if device_info is None:
             device_info = InternalPcieDevice.scan_devices()[0]
@@ -1235,8 +1300,8 @@ class HailoFormatFlags(_pyhailort.FormatFlags):
 
 SUPPORTED_PROTOCOL_VERSION = 2
 SUPPORTED_FW_MAJOR = 4
-SUPPORTED_FW_MINOR = 12
+SUPPORTED_FW_MINOR = 13
-SUPPORTED_FW_REVISION = 1
+SUPPORTED_FW_REVISION = 0
 
 MEGA_MULTIPLIER = 1000.0 * 1000.0
 
@@ -1245,7 +1310,7 @@ class DeviceArchitectureTypes(IntEnum):
     HAILO8_A0 = 0
     HAILO8 = 1
     HAILO8L = 2
-    MERCURY_CA = 3
+    HAILO15 = 3
 
     def __str__(self):
         return self.name
@@ -1301,8 +1366,8 @@ class BoardInformation(object):
         if ((device_arch == DeviceArchitectureTypes.HAILO8) or
             (device_arch == DeviceArchitectureTypes.HAILO8L)):
             return 'hailo8'
-        elif device_arch == DeviceArchitectureTypes.MERCURY_CA:
+        elif device_arch == DeviceArchitectureTypes.HAILO15:
-            return 'mercury'
+            return 'hailo15'
         else:
             raise HailoRTException("Unsupported device architecture.")
 
@@ -1549,7 +1614,6 @@ class SupportedFeatures(object):
     def _is_feature_enabled(self, feature):
         return (self.supported_features & feature) != 0
 
-
 class Control:
     """The control object of this device, which implements the control API of the Hailo device.
     Should be used only from Device.control"""
@@ -1557,7 +1621,7 @@ class Control:
     WORD_SIZE = 4
 
     def __init__(self, device: '_pyhailort.Device'):
-        self._device = device
+        self.__device = device
         self._logger = default_logger()
 
         # TODO: should remove?
@@ -1566,12 +1630,19 @@ class Control:
 
         self._identify_info = self.identify()
 
+    @property
+    def _device(self):
+        if not self.__device.is_valid():
+            raise HailoRTInvalidOperationException("The device in use has been released. "
+                "This can happen if 'device.release()' has been called, or one-liner usage of control 'Device().control.XX()'")
+        return self.__device
+
     @property
     def device_id(self):
         """Getter for the device_id.
 
         Returns:
-            str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Core" for core devices.
+            str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Integrated" for integrated nnc devices.
         """
         return self._device.device_id
 
@@ -1672,9 +1743,7 @@ class Control:
             measurement types, please look at
             :class:`~hailo_platform.pyhailort.pyhailort.PowerMeasurementTypes`.
         """
-        if ((self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8) and
+
-            (self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8L)):
-            raise HailoRTException("Invalid device architecture: {}".format(self._identify_info.device_architecture))
         with ExceptionWrapper():
             return self._device.power_measurement(dvm, measurement_type)
 
@@ -1751,9 +1820,7 @@ class Control:
             For all supported measurement types view
             :class:`~hailo_platform.pyhailort.pyhailort.PowerMeasurementTypes`.
         """
-        if ((self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8) and
+
-            (self._identify_info.device_architecture != DeviceArchitectureTypes.HAILO8L)):
-            raise HailoRTException("Invalid device architecture: {}".format(self._identify_info.device_architecture))
         with ExceptionWrapper():
             return self._device.get_power_measurement(buffer_index, should_clear)
 
@@ -2181,7 +2248,7 @@ class Control:
 
 
 class Device:
-    """ Hailo device object representation. """
+    """ Hailo device object representation (for inference use VDevice)"""
 
     @classmethod
     def scan(cls):
@@ -2246,7 +2313,7 @@ class Device:
         """Getter for the device_id.
 
         Returns:
-            str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Core" for core devices.
+            str: A string ID of the device. BDF for PCIe devices, IP address for Ethernet devices, "Integrated" for integrated nnc devices.
         """
         return self._device_id
 
@@ -2258,7 +2325,7 @@ class Device:
             configure_params_by_name (dict, optional): Maps between each net_group_name to configure_params. If not provided, default params will be applied
         """
         if self._creation_pid != os.getpid():
-            raise HailoRTException("VDevice can only be configured from the process it was created in.")
+            raise HailoRTException("Device can only be configured from the process it was created in.")
         with ExceptionWrapper():
             configured_apps = self._device.configure(hef._hef, configure_params_by_name)
         configured_networks = [ConfiguredNetwork(configured_app, self, hef) for configured_app in configured_apps]
@@ -2315,9 +2382,10 @@ class VDevice(object):
             params (:obj:`hailo_platform.pyhailort.pyhailort.VDeviceParams`, optional): VDevice params, call
                 :func:`VDevice.create_params` to get default params. Excludes 'device_ids'.
             device_ids (list of str, optional): devices ids to create VDevice from, call :func:`Device.scan` to get
-                list of all available devices. Excludes 'params'.
+                list of all available devices. Excludes 'params'. Cannot be used together with device_id.
         """
         gc.collect()
+        self._logger = default_logger()
 
         # VDevice __del__ function tries to release self._vdevice.
         # to avoid AttributeError if the __init__ func fails, we set it to None first.
@@ -2330,21 +2398,37 @@ class VDevice(object):
         self._creation_pid = os.getpid()
 
         self._device_ids = device_ids
-        if self._device_ids is not None:
-            if self._params is not None:
-                raise HailoRTException("VDevice can be created from params or device ids. Both parameters were passed to the c'tor")
 
         self._open_vdevice()
 
+    def _before_fork(self):
+        if self._vdevice is not None:
+            self._vdevice.before_fork()
+            for configured_network in self._loaded_network_groups:
+                configured_network._before_fork()
+
+    def _after_fork_in_parent(self):
+        if self._vdevice is not None:
+            self._vdevice.after_fork_in_parent()
+            for configured_network in self._loaded_network_groups:
+                configured_network._after_fork_in_parent()
+
+    def _after_fork_in_child(self):
+        if self._vdevice is not None:
+            self._vdevice.after_fork_in_child()
+            for configured_network in self._loaded_network_groups:
+                configured_network._after_fork_in_child()
+
     def _open_vdevice(self):
-        if self._device_ids is not None:
-            with ExceptionWrapper():
-                self._vdevice = _pyhailort.VDevice.create_from_ids(self._device_ids)
-        else:
         if self._params is None:
             self._params = VDevice.create_params()
+        if  sys.platform != "win32" and self._params.multi_process_service:
+            os.register_at_fork(before=lambda: self._before_fork())
+            os.register_at_fork(after_in_parent=lambda: self._after_fork_in_parent())
+            os.register_at_fork(after_in_child=lambda: self._after_fork_in_child())
         with ExceptionWrapper():
-                self._vdevice = _pyhailort.VDevice.create(self._params)
+            device_ids = [] if self._device_ids is None else self._device_ids
+            self._vdevice = _pyhailort.VDevice.create(self._params, device_ids)
 
     def __enter__(self):
         return self
@@ -2496,7 +2580,7 @@ class OutputVStreamParams(object):
     @staticmethod
     def make(configured_network, quantized=True, format_type=None, timeout_ms=None, queue_size=None, network_name=None):
         """Create output virtual stream params from a configured network group. These params determine the format of the
-        data that will be fed into the network group.
+        data that will be returned from the network group.
 
         Args:
             configured_network (:class:`ConfiguredNetwork`): The configured network group for which
@@ -2537,12 +2621,12 @@ class OutputVStreamParams(object):
     @staticmethod
     def make_from_network_group(configured_network, quantized=True, format_type=None, timeout_ms=None, queue_size=None, network_name=None):
         """Create output virtual stream params from a configured network group. These params determine the format of the
-        data that will be fed into the network group.
+        data that will be returned from the network group.
 
         Args:
             configured_network (:class:`ConfiguredNetwork`): The configured network group for which
                 the params are created.
-            quantized (bool): Whether the data fed into the chip is already quantized. True means
+            quantized (bool): Whether the data returned from the chip is already quantized. True means
                 the data is already quantized. False means it's HailoRT's responsibility to quantize
                 (scale) the data. Defaults to True.
             format_type (:class:`~hailo_platform.pyhailort.pyhailort.FormatType`): The
@@ -2566,12 +2650,12 @@ class OutputVStreamParams(object):
     @staticmethod
     def make_groups(configured_network, quantized=True, format_type=None, timeout_ms=None, queue_size=None):
         """Create output virtual stream params from a configured network group. These params determine the format of the
-        data that will be fed into the network group. The params groups are splitted with respect to their underlying streams for multi process usges.
+        data that will be returned from the network group. The params groups are splitted with respect to their underlying streams for multi process usges.
 
         Args:
             configured_network (:class:`ConfiguredNetwork`): The configured network group for which
                 the params are created.
-            quantized (bool): Whether the data fed into the chip is already quantized. True means
+            quantized (bool): Whether the data returned from the chip is already quantized. True means
                 the data is already quantized. False means it's HailoRT's responsibility to quantize
                 (scale) the data. Defaults to True.
             format_type (:class:`~hailo_platform.pyhailort.pyhailort.FormatType`): The
@@ -2660,6 +2744,19 @@ class InputVStream(object):
         with ExceptionWrapper():
             return self._send_object.info
 
+    def _before_fork(self):
+        if self._send_object is not None:
+            self._send_object.before_fork()
+
+    def _after_fork_in_parent(self):
+        if self._send_object is not None:
+            self._send_object.after_fork_in_parent()
+
+    def _after_fork_in_child(self):
+        if self._send_object is not None:
+            self._send_object.after_fork_in_child()
+
+
 class InputVStreams(object):
     """Input vstreams pipelines that allows to send data, to be used as a context manager."""
 
@@ -2709,6 +2806,19 @@ class InputVStreams(object):
     def __iter__(self):
         return iter(self._vstreams.values())
 
+    def _before_fork(self):
+        for vstream in self._vstreams.values():
+            vstream._before_fork()
+
+    def _after_fork_in_parent(self):
+        for vstream in self._vstreams.values():
+            vstream._after_fork_in_parent()
+
+    def _after_fork_in_child(self):
+        for vstream in self._vstreams.values():
+            vstream._after_fork_in_child()
+
+
 class OutputLayerUtils(object):
     def __init__(self, hef, vstream_name, pipeline, net_group_name=""):
         self._hef = hef
@@ -2833,6 +2943,19 @@ class OutputVStream(object):
         with ExceptionWrapper():
             return self._recv_object.info
 
+    def _before_fork(self):
+        if self._recv_object is not None:
+            self._recv_object.before_fork()
+
+    def _after_fork_in_parent(self):
+        if self._recv_object is not None:
+            self._recv_object.after_fork_in_parent()
+
+    def _after_fork_in_child(self):
+        if self._recv_object is not None:
+            self._recv_object.after_fork_in_child()
+
+
 class OutputVStreams(object):
     """Output virtual streams pipelines that allows to receive data, to be used as a context manager."""
 
@@ -2896,13 +3019,26 @@ class OutputVStreams(object):
     def __iter__(self):
         return iter(self._vstreams.values())
 
-class YOLOv5PostProcessingOp(object):
+    def _before_fork(self):
+        for vstream in self._vstreams.values():
+            vstream._before_fork()
+
+    def _after_fork_in_parent(self):
+        for vstream in self._vstreams.values():
+            vstream._after_fork_in_parent()
+
+    def _after_fork_in_child(self):
+        for vstream in self._vstreams.values():
+            vstream._after_fork_in_child()
+
+
+class YOLOv5PostProcessOp(object):
 
     def __init__(self, anchors, shapes, formats, quant_infos, image_height, image_width, confidence_threshold, iou_threshold, num_of_classes,
-            should_dequantize, max_boxes, should_sigmoid, one_class_per_bbox=True):
+            max_boxes, cross_classes=True):
 
-        self._op = _pyhailort.YOLOv5PostProcessingOp.create(anchors, shapes, formats, quant_infos, image_height, image_width, confidence_threshold,
+        self._op = _pyhailort.YOLOv5PostProcessOp.create(anchors, shapes, formats, quant_infos, image_height, image_width, confidence_threshold,
-            iou_threshold, num_of_classes, should_dequantize, max_boxes, should_sigmoid, one_class_per_bbox)
+            iou_threshold, num_of_classes, max_boxes, cross_classes)
 
     def execute(self, net_flow_tensors):
         return self._op.execute(net_flow_tensors)

hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/hailocli_commands.py

@@ -1,16 +1,18 @@
 import os
-import subprocess
 import pathlib
+import subprocess
 import sys
 
+import pkg_resources
+
 import hailo_platform
 from hailo_platform.tools.hailocli.base_utils import HailortCliUtil
-import pkg_resources
 
 """
     HailoRTCLI matching commands in Hailo-CLI tool.
 """
 
+
 class BenchmarkCommandCLI(HailortCliUtil):
     def __init__(self, parser):
         super().__init__(parser, 'benchmark')
@@ -18,12 +20,14 @@ class BenchmarkCommandCLI(HailortCliUtil):
 
 class FWConfigCommandCLI(HailortCliUtil):
     """CLI tool for changing the FW configuration (User Config)"""
+
     def __init__(self, parser):
         super().__init__(parser, 'fw-config')
 
 
 class BoardConfigCommandCLI(HailortCliUtil):
     """CLI tool for changing the FW configuration (Board Config)"""
+
     def __init__(self, parser):
         super().__init__(parser, 'board-config')
 
@@ -60,34 +64,30 @@ class SensorConfigCommandCLI(HailortCliUtil):
 
 class FWUpdaterCLI(HailortCliUtil):
     """Cli tool for firmware updates"""
+
     def __init__(self, parser):
         super().__init__(parser, 'fw-update')
 
 
 class SSBUpdaterCLI(HailortCliUtil):
     """Cli tool for second stage boot updates"""
+
     def __init__(self, parser):
         super().__init__(parser, 'ssb-update')
 
 
 class UDPRateLimiterCLI(HailortCliUtil):
     """CLI tool for UDP rate limitation."""
+
     def __init__(self, parser):
         super().__init__(parser, 'udp-rate-limiter')
 
 
-class VersionCLI(HailortCliUtil):
-    """CLI tool for hailort version."""
-    def __init__(self, parser):
-        super().__init__(parser, '--version')
-
-
 class TutorialRequired(Exception):
     pass
 
 
 class TutorialRunnerCLI():
-
     TUTORIALS_DIR = os.path.join(pathlib.Path(hailo_platform.__file__).parent.parent, 'hailo_tutorials/notebooks/')
     TUTORIALS_REQUIREMENTS = ["jupyter"]
     ERROR_MSG = """

hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/main.py

@@ -1,14 +1,19 @@
 #!/usr/bin/env python
 
 import argparse
-import argcomplete
 import sys
 
+import argcomplete
+
 import hailo_platform
-from hailo_platform.tools.hailocli.base_utils import HailortCliUtil, Helper, HailortCliUtilError
+from hailo_platform.tools.hailocli.base_utils import HailortCliUtil, HailortCliUtilError, Helper
-from hailo_platform.tools.hailocli.hailocli_commands import (FWUpdaterCLI, SSBUpdaterCLI, ControlCommandCLI, ScanCommandCLI,
+from hailo_platform.tools.hailocli.hailocli_commands import (BenchmarkCommandCLI, ControlCommandCLI, FWConfigCommandCLI,
-                                                            LoggerCommandCLI, MeasurePowerCommandCLI, RunCommandCLI, SensorConfigCommandCLI,
+                                                             FWUpdaterCLI, LoggerCommandCLI, MeasurePowerCommandCLI,
-                                                            FWConfigCommandCLI, BenchmarkCommandCLI, UDPRateLimiterCLI, MonitorCommandCLI, ParseHEFCommandCLI, TutorialRunnerCLI)
+                                                             MonitorCommandCLI, ParseHEFCommandCLI, RunCommandCLI,
+                                                             SSBUpdaterCLI, ScanCommandCLI, SensorConfigCommandCLI,
+                                                             TutorialRunnerCLI, UDPRateLimiterCLI)
+from hailo_platform.tools.hailocli.version_action import CustomVersionAction
+
 
 # Note: PlatformCommands are external dependencies in phase2-sdk/demos repo; don't change!
 class PlatformCommands:
@@ -25,15 +30,16 @@ class PlatformCommands:
         'sensor-config': ('Sensor configuration tool', SensorConfigCommandCLI),
         'run': ('Run a compiled network', RunCommandCLI),
         'benchmark': ('Measure basic performance on compiled network', BenchmarkCommandCLI),
-        'monitor': ("Monitor of networks - Presents information about the running networks. To enable monitor, set in the application process the environment variable 'SCHEDULER_MONITOR' to 1.", MonitorCommandCLI),
+        'monitor': ("Monitor of networks - Presents information about the running networks. To enable monitor, set in the application process the environment variable 'HAILO_MONITOR' to 1.", MonitorCommandCLI),
         'parse-hef': (' Parse HEF to get information about its components', ParseHEFCommandCLI),
         'measure-power': ('Measures power consumption', MeasurePowerCommandCLI),
         'tutorial': ('Runs the tutorials in jupyter notebook', TutorialRunnerCLI),
-        #'--version': ('Print program version and exit', VersionCLI)
     }
 
     def __init__(self):
         self.parser = argparse.ArgumentParser(description=self._get_generic_description())
+        self.parser.register('action', 'custom_version', CustomVersionAction)
+        self.parser.add_argument('--version', action='custom_version')
         self.subparsers = self.parser.add_subparsers(help='Hailo utilities aimed to help with everything you need')
         self.COMMANDS = {}
         self.COMMANDS.update(type(self).PLATFORM_COMMANDS)
@@ -60,6 +66,7 @@ class PlatformCommands:
     # Dependency injection for testing
     def _run(self, argv):
         self.COMMANDS['help'] = ('show the list of commands', Helper(self.COMMANDS))
+
         # Create the commands and let them set the arguments
         commands = {}
         for command_name, (help_, command_class) in self.COMMANDS.items():
@@ -79,10 +86,10 @@ class PlatformCommands:
         command_name = argv[0]
         if (command_name in commands) and isinstance(commands[command_name], HailortCliUtil):
             # HailortCliUtil just passes the rest of the argv to hailortcli
-            try :
+            try:
                 return commands[command_name].run(argv[1:])
             except HailortCliUtilError as e:
-                print('\n'+ str(e))
+                print('\n' + str(e))
                 return
 
         # This isn't a HailortCliUtil commnad, parse with argparse

hailort/libhailort/bindings/python/platform/hailo_platform/tools/hailocli/version_action.py

@@ -0,0 +1,31 @@
+import argparse
+
+import hailo_platform
+
+
+class CustomVersionAction(argparse.Action):
+    def __init__(self,
+                 option_strings,
+                 dest=argparse.SUPPRESS,
+                 default=argparse.SUPPRESS,
+                 help="show program's version number and exit"):
+        super(CustomVersionAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    @staticmethod
+    def _print_version():
+        print(f'HailoRT v{hailo_platform.__version__}')
+
+        try:
+            import hailo_sdk_client
+            print(f'Hailo Dataflow Compiler v{hailo_sdk_client.__version__}')
+        except ImportError:
+            pass
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        self._print_version()
+        parser.exit()

hailort/libhailort/bindings/python/platform/hailo_platform/tools/udp_rate_limiter.py

@@ -36,7 +36,7 @@ def get_max_supported_kbps(hw_arch="hailo8"):
 
 class RateLimiterWrapper(object):
     """UDPRateLimiter wrapper enabling ``with`` statements."""
-    def __init__(self, network_group, fps=1, fps_factor=1.0):
+    def __init__(self, network_group, fps=1, fps_factor=1.0, remote_ip=None, hw_arch=None):
         """RateLimiterWrapper constructor.
 
         Args:
@@ -48,10 +48,18 @@ class RateLimiterWrapper(object):
         if not isinstance(network_group, ConfiguredNetwork):
             return RateLimiterException("The API was changed. RateLimiterWrapper accept ConfiguredNetwork instead of ActivatedNetwork")
         self._network_group = network_group
-        self._remote_ip = network_group._target.remote_ip
+        if remote_ip is not None:
+            self._remote_ip = remote_ip
+        else:
+            # this line should be removed. this parameter will be removed from the object
+            self._remote_ip = network_group._target.device_id
         self._fps = fps
         self._fps_factor = fps_factor
-        self._hw_arch = network_group._target._hw_arch
+        if hw_arch is not None:
+            self._hw_arch = hw_arch
+        else:
+            # this line should be removed. this parameter will be removed from the object
+            self._hw_arch = network_group._target._hw_arch if hasattr(network_group._target, '_hw_arch') else None
         self._rates_dict = {}
         self._tc_dict = {}
 

hailort/libhailort/bindings/python/platform/hailo_tutorials/notebooks/HRT_0_Inference_Tutorial.ipynb

@@ -43,12 +43,12 @@
    "source": [
     "import numpy as np\n",
     "from multiprocessing import Process\n",
-    "from hailo_platform import (HEF, PcieDevice, HailoStreamInterface, InferVStreams, ConfigureParams,\n",
+    "from hailo_platform import (HEF, VDevice, HailoStreamInterface, InferVStreams, ConfigureParams,\n",
     "    InputVStreamParams, OutputVStreamParams, InputVStreams, OutputVStreams, FormatType)\n",
     "\n",
     "# The target can be used as a context manager (\"with\" statement) to ensure it's released on time.\n",
     "# Here it's avoided for the sake of simplicity\n",
-    "target = PcieDevice()\n",
+    "target = VDevice()\n",
     "\n",
     "# Loading compiled HEFs to device:\n",
     "model_name = 'resnet_v1_18'\n",

hailort/libhailort/bindings/python/platform/setup.py

@@ -69,6 +69,6 @@ if __name__ == "__main__":
             "linux_aarch64",
         ],
         url="https://hailo.ai/",
-        version="4.12.1",
+        version="4.13.0",
         zip_safe=False,
     )

hailort/libhailort/bindings/python/src/CMakeLists.txt

@@ -24,6 +24,7 @@ pybind11_add_module(_pyhailort
     hef_api.cpp
     vstream_api.cpp
     quantization_api.cpp
+    ${HAILORT_OPS_CPP_SOURCES}
     ${HAILORT_COMMON_CPP_SOURCES}
 )
 
@@ -44,6 +45,10 @@ target_link_libraries(_pyhailort PRIVATE libhailort spdlog::spdlog)
 if(WIN32)
     target_link_libraries(_pyhailort PRIVATE Ws2_32 Iphlpapi Shlwapi)
 endif()
+if(HAILO_BUILD_SERVICE)
+    target_link_libraries(_pyhailort PRIVATE grpc++_unsecure hailort_rpc_grpc_proto hef_proto)
+endif()
+
 target_compile_options(_pyhailort PRIVATE ${HAILORT_COMPILE_OPTIONS})
 exclude_archive_libs_symbols(_pyhailort)
 

hailort/libhailort/bindings/python/src/bindings_common.hpp

@@ -13,9 +13,14 @@
 #include "hailo/hailort.h"
 #include "hailo/hailort_common.hpp"
 #include "hailo/network_group.hpp"
-#include "utils.hpp"
+
 #include "common/logger_macros.hpp"
 
+#include "utils.hpp"
+
+#include <pybind11/numpy.h>
+
+
 namespace hailort
 {
 class HailoRTBindingsCommon

hailort/libhailort/bindings/python/src/device_api.cpp

@@ -126,7 +126,7 @@ bool DeviceWrapper::get_overcurrent_state()
 py::bytes DeviceWrapper::read_memory(uint32_t address, uint32_t length)
 {
     std::unique_ptr<std::string> response = make_unique_nothrow<std::string>(length, '\x00');
-    VALIDATE_NOT_NULL(response);
+    VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY);
 
     MemoryView data_view(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(response->data())), length);
     auto status = device().read_memory(address, data_view);
@@ -150,7 +150,7 @@ void DeviceWrapper::test_chip_memories()
 void DeviceWrapper::i2c_write(hailo_i2c_slave_config_t *slave_config, uint32_t register_address, py::bytes data,
     uint32_t length)
 {
-    VALIDATE_NOT_NULL(slave_config);
+    VALIDATE_NOT_NULL(slave_config, HAILO_INVALID_ARGUMENT);
 
     std::string data_str(data);
     MemoryView data_view = MemoryView::create_const(data_str.c_str(), length);
@@ -160,10 +160,10 @@ void DeviceWrapper::i2c_write(hailo_i2c_slave_config_t *slave_config, uint32_t r
 
 py::bytes DeviceWrapper::i2c_read(hailo_i2c_slave_config_t *slave_config, uint32_t register_address, uint32_t length)
 {
-    VALIDATE_NOT_NULL(slave_config);
+    VALIDATE_NOT_NULL(slave_config, HAILO_INVALID_ARGUMENT);
 
     std::unique_ptr<std::string> response = make_unique_nothrow<std::string>(length, '\x00');
-    VALIDATE_NOT_NULL(response);
+    VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY);
 
     MemoryView data_view(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(response->data())), length);
     auto status = device().i2c_read(*slave_config, register_address, data_view);
@@ -231,7 +231,7 @@ py::bytes DeviceWrapper::read_user_config()
 
     std::unique_ptr<std::string> response = make_unique_nothrow<std::string>(
         const_cast<char*>(reinterpret_cast<const char*>(config_buffer->data())), config_buffer->size());
-    VALIDATE_NOT_NULL(response);
+    VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY);
 
     return *response;
 }
@@ -257,7 +257,7 @@ py::bytes DeviceWrapper::read_board_config()
 
     std::unique_ptr<std::string> response = make_unique_nothrow<std::string>(
         const_cast<char*>(reinterpret_cast<const char*>(config_buffer->data())), config_buffer->size());
-    VALIDATE_NOT_NULL(response);
+    VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY);
     
     return *response;
 }
@@ -309,7 +309,7 @@ py::bytes DeviceWrapper::sensor_get_sections_info()
     
     std::unique_ptr<std::string> response = make_unique_nothrow<std::string>(
         const_cast<char*>(reinterpret_cast<const char*>(buffer->data())), buffer->size());
-    VALIDATE_NOT_NULL(response);
+    VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY);
 
     return *response;
 }
@@ -446,7 +446,7 @@ void DeviceWrapper::direct_write_memory(uint32_t address, py::bytes buffer)
 {
     const auto buffer_str = static_cast<std::string>(buffer);
     hailo_status status = device().direct_write_memory(address, buffer_str.c_str(),
-        (uint32_t) (buffer_str.length()));
+        (uint32_t)(buffer_str.length()));
     VALIDATE_STATUS(status);
 }
 
@@ -478,6 +478,8 @@ void DeviceWrapper::set_sleep_state(hailo_sleep_state_t sleep_state)
 void DeviceWrapper::add_to_python_module(py::module &m)
 {
     py::class_<DeviceWrapper>(m, "Device")
+    .def("is_valid", &DeviceWrapper::is_valid)
+
     // Scan
     .def("scan", &DeviceWrapper::scan)
 

hailort/libhailort/bindings/python/src/device_api.hpp

@@ -11,13 +11,15 @@
 #ifndef _DEVICE_API_HPP_
 #define _DEVICE_API_HPP_
 
-#include "utils.hpp"
+#include "hailo/hailort.h"
-#include "hailo/hailort.hpp"
+#include "hailo/device.hpp"
+
 #include "common/socket.hpp"
+
+#include "utils.hpp"
 #include "hef_api.hpp"
 
 #include <pybind11/pybind11.h>
-#include <pybind11/pybind11.h>
 
 
 namespace hailort
@@ -53,16 +55,21 @@ public:
 
     Device& device()
     {
-        VALIDATE_NOT_NULL(m_device);
+        VALIDATE_NOT_NULL(m_device, HAILO_INTERNAL_FAILURE);
         return *(m_device.get());
     }
 
     const Device& device() const
     {
-        VALIDATE_NOT_NULL(m_device);
+        VALIDATE_NOT_NULL(m_device, HAILO_INTERNAL_FAILURE);
         return *(m_device.get());
     }
 
+    bool is_valid()
+    {
+        return (nullptr != m_device);
+    }
+
     Device& operator*() // Used for control_internals
     {
         return device();

hailort/libhailort/bindings/python/src/hef_api.cpp

@@ -207,9 +207,10 @@ ActivatedAppContextManagerWrapper::ActivatedAppContextManagerWrapper(ConfiguredN
 const ActivatedNetworkGroup& ActivatedAppContextManagerWrapper::enter()
 {
     auto activated = m_net_group.activate(m_network_group_params);
+    if (activated.status() != HAILO_NOT_IMPLEMENTED) {
         VALIDATE_EXPECTED(activated);
-
         m_activated_net_group = activated.release();
+    }
 
     return std::ref(*m_activated_net_group);
 }
@@ -273,7 +274,9 @@ void HefWrapper::initialize_python_module(py::module &m)
         .def("wait_for_activation", [](ConfiguredNetworkGroup& self, uint32_t timeout_ms)
             {
                 auto status = self.wait_for_activation(std::chrono::milliseconds(timeout_ms));
+                if (status != HAILO_NOT_IMPLEMENTED) {
                     VALIDATE_STATUS(status);
+                }
             })
         .def("InputVStreams", [](ConfiguredNetworkGroup &self, std::map<std::string, hailo_vstream_params_t> &input_vstreams_params)
             {
@@ -295,6 +298,49 @@ void HefWrapper::initialize_python_module(py::module &m)
 
             return py::cast(results.value());
         })
+        .def("before_fork", [](ConfiguredNetworkGroup& self)
+        {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+            auto status = self.before_fork();
+            VALIDATE_STATUS(status);
+#else
+            (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+        })
+        .def("after_fork_in_parent", [](ConfiguredNetworkGroup& self)
+        {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+            auto status = self.after_fork_in_parent();
+            VALIDATE_STATUS(status);            
+#else
+            (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+        })
+        .def("after_fork_in_child", [](ConfiguredNetworkGroup& self)
+        {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+            auto status = self.after_fork_in_child();
+            VALIDATE_STATUS(status);
+#else
+            (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+        })
+        .def("set_scheduler_timeout", [](ConfiguredNetworkGroup& self, int timeout, const std::string &network_name="")
+        {
+            auto timeout_mili = std::chrono::milliseconds(timeout);
+            auto status = self.set_scheduler_timeout(timeout_mili, network_name);
+            VALIDATE_STATUS(status);
+        })
+        .def("set_scheduler_threshold", [](ConfiguredNetworkGroup& self, uint32_t threshold)
+        {
+            auto status = self.set_scheduler_threshold(threshold);
+            VALIDATE_STATUS(status);
+        })
+        .def("set_scheduler_priority", [](ConfiguredNetworkGroup& self, uint8_t priority)
+        {
+            auto status = self.set_scheduler_priority(priority);
+            VALIDATE_STATUS(status);
+        })
         ;
 
     ActivatedAppContextManagerWrapper::add_to_python_module(m);

hailort/libhailort/bindings/python/src/internal/CMakeLists.txt

@@ -1,11 +1,13 @@
-cmake_minimum_required(VERSION 3.0.0)
+cmake_minimum_required(VERSION 3.15.0)
 
 pybind11_add_module(_pyhailort_internal SHARED
     pyhailort_internal.cpp
     control_api.cpp
-    ${HAILORT_SRCS_ABS}
+    $<TARGET_OBJECTS:libhailort>
 )
 
+add_dependencies(_pyhailort_internal libhailort)
+
 set_target_properties(_pyhailort_internal PROPERTIES
     CXX_STANDARD              14
     CXX_STANDARD_REQUIRED     YES
@@ -22,7 +24,6 @@ target_include_directories(_pyhailort_internal
 )
 
 target_link_libraries(_pyhailort_internal PRIVATE
-    libhailort
     hef_proto
     spdlog::spdlog
     readerwriterqueue

hailort/libhailort/bindings/python/src/internal/control_api.cpp

@@ -93,7 +93,7 @@ void ControlWrapper::validate_firmware_update(DeviceWrapper &device, py::bytes m
 py::bytes ControlWrapper::sensor_get_config(DeviceWrapper &device, uint32_t section_index, uint32_t offset, uint32_t data_length)
 {
     std::unique_ptr<std::string> response = make_unique_nothrow<std::string>(data_length, '\x00');
-    VALIDATE_NOT_NULL(response);
+    VALIDATE_NOT_NULL(response, HAILO_OUT_OF_HOST_MEMORY);
     
     auto status = Control::sensor_get_config(*device, section_index, offset, data_length, (uint8_t*)(response->data()));
     VALIDATE_STATUS(status);

hailort/libhailort/bindings/python/src/internal/control_api.hpp

@@ -11,7 +11,7 @@
 #ifndef _CONTROL_API_HPP_
 #define _CONTROL_API_HPP_
 
-#include "control.hpp"
+#include "device_common/control.hpp"
 #include "utils.hpp"
 
 #include "device_api.hpp"

hailort/libhailort/bindings/python/src/internal/pyhailort_internal.cpp

@@ -1,3 +1,15 @@
+
+
+#include "hailo/hailort.h"
+
+#include "transform/transform_internal.hpp"
+#include "bindings_common.hpp"
+
+#include "pyhailort_internal.hpp"
+#include "control_api.hpp"
+#include "utils.hpp"
+#include "utils.h"
+
 #include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
 #include <pybind11/detail/common.h>
@@ -6,22 +18,15 @@
 #include <pybind11/functional.h>
 #include <vector>
 
-#include "pyhailort_internal.hpp"
-#include "control_api.hpp"
-#include "utils.hpp"
-#include "utils.h"
-
-#include "hailo/hailort.h"
-#include "transform_internal.hpp"
-#include "bindings_common.hpp"
-
 
 namespace hailort
 {
+// TODO: Remove (HRT-9944)
+// Duplicated for hailo post process test with python API.
+static const uint32_t TEST_NUM_OF_CLASSES = 80;
 
-static const uint32_t TEST_NUM_OF_CLASSES2 = 80;
 
-py::array PyhailortInternal::get_yolov5_post_process_expected_buffer()
+Expected<Buffer> get_expected_buffer_float32()
 {
     static const uint32_t DETECTION_CLASS_ID_1 = 0;
     static const float32_t CLASS_ID_1_DETECTION_COUNT = 5;
@@ -105,13 +110,13 @@ py::array PyhailortInternal::get_yolov5_post_process_expected_buffer()
     };
 
     static const uint32_t DETECTION_COUNT = 9;
-    auto buffer_size = (DETECTION_COUNT * sizeof(hailo_bbox_float32_t)) + (TEST_NUM_OF_CLASSES2 * sizeof(float32_t));
+    auto buffer_size = (DETECTION_COUNT * sizeof(hailo_bbox_float32_t)) + (TEST_NUM_OF_CLASSES * sizeof(float32_t));
-    auto buffer_expected = hailort::Buffer::create(buffer_size, 0);
+    auto buffer_expected = Buffer::create(buffer_size, 0);
-    // CATCH_REQUIRE_EXPECTED(buffer_expected);
+    CHECK_EXPECTED(buffer_expected);
     auto buffer = buffer_expected.release();
 
     size_t offset = 0;
-    for (uint32_t class_index = 0; class_index < TEST_NUM_OF_CLASSES2; class_index++) {
+    for (uint32_t class_index = 0; class_index < TEST_NUM_OF_CLASSES; class_index++) {
         if (DETECTION_CLASS_ID_1 == class_index) {
             memcpy(buffer.data() + offset, &CLASS_ID_1_DETECTION_COUNT, sizeof(CLASS_ID_1_DETECTION_COUNT));
             offset += sizeof(CLASS_ID_1_DETECTION_COUNT);
@@ -160,12 +165,20 @@ py::array PyhailortInternal::get_yolov5_post_process_expected_buffer()
         }
     }
 
+    return buffer;
+}
+
+py::array PyhailortInternal::get_yolov5_post_process_expected_buffer()
+{
+    auto buffer = get_expected_buffer_float32();
+    VALIDATE_EXPECTED(buffer);
+
     // Note: The ownership of the buffer is transferred to Python wrapped as a py::array.
     //       When the py::array isn't referenced anymore in Python and is destructed, the py::capsule's dtor
     //       is called too (and it deletes the raw buffer)
     auto type = py::dtype(HailoRTBindingsCommon::convert_format_type_to_string(HAILO_FORMAT_TYPE_FLOAT32));
-    auto shape = *py::array::ShapeContainer({buffer.size()});
+    auto shape = *py::array::ShapeContainer({buffer->size()});
-    const auto unmanaged_addr = buffer.release();
+    const auto unmanaged_addr = buffer.release().release();
     return py::array(type, shape, unmanaged_addr,
         py::capsule(unmanaged_addr, [](void *p) { delete reinterpret_cast<uint8_t*>(p); }));
 }
@@ -261,10 +274,10 @@ bool PyhailortInternal::is_output_transformation_required(
 
 py::list PyhailortInternal::get_all_layers_info(const HefWrapper &hef, const std::string &net_group_name)
 {
-    auto network_group_metadata = hef.hef_ptr()->pimpl->get_network_group_metadata(net_group_name);
+    auto core_op_metadata = hef.hef_ptr()->pimpl->get_core_op_metadata(net_group_name);
-    VALIDATE_EXPECTED(network_group_metadata);
+    VALIDATE_EXPECTED(core_op_metadata);
 
-    return py::cast(network_group_metadata->get_all_layer_infos());
+    return py::cast(core_op_metadata->get_all_layer_infos());
 }
 
 PYBIND11_MODULE(_pyhailort_internal, m) {

hailort/libhailort/bindings/python/src/internal/pyhailort_internal.hpp

@@ -10,6 +10,11 @@
 #ifndef _PYHAILORT_INTERNAL_
 #define _PYHAILORT_INTERNAL_
 
+#include "hef/hef_internal.hpp"
+
+#include "hef_api.hpp"
+#include "utils.hpp"
+#include "utils.h"
 #include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
 #include <pybind11/detail/common.h>
@@ -18,10 +23,6 @@
 #include <pybind11/functional.h>
 #include <vector>
 
-#include "hef_internal.hpp"
-#include "hef_api.hpp"
-#include "utils.hpp"
-#include "utils.h"
 
 namespace hailort
 {

hailort/libhailort/bindings/python/src/net_flow_api.hpp

@@ -10,42 +10,70 @@
 #ifndef _HAILO_NET_FLOW_API_HPP_
 #define _HAILO_NET_FLOW_API_HPP_
 
+#include "hailo/hailort.h"
+
+#include "net_flow/ops/yolo_post_process.hpp"
+
 #include "utils.hpp"
-#include "hailo/hailort.hpp"
 #include "bindings_common.hpp"
-#include "net_flow/ops/yolo_post_processing.hpp"
+
 
 namespace hailort
 {
 namespace net_flow
 {
 
-class YOLOv5PostProcessingOpWrapper
+class YOLOv5PostProcessOpWrapper
 {
 public:
-    static YOLOv5PostProcessingOpWrapper create(const std::vector<std::vector<int>> &anchors,
+    static YOLOv5PostProcessOpWrapper create(const std::vector<std::vector<int>> &anchors,
         const std::vector<hailo_3d_image_shape_t> &shapes, const std::vector<hailo_format_t> &formats,
         const std::vector<hailo_quant_info_t> &quant_infos, float32_t image_height, float32_t image_width, float32_t confidence_threshold,
-        float32_t iou_threshold, uint32_t num_of_classes, bool should_dequantize, uint32_t max_boxes, bool should_sigmoid,
+        float32_t iou_threshold, uint32_t num_of_classes, uint32_t max_boxes,
-        bool one_class_per_bbox=true)
+        bool cross_classes=true)
     {
-        auto op = YOLOv5PostProcessingOp::create(anchors, shapes, formats, quant_infos, image_height, image_width,
+        std::map<std::string, net_flow::BufferMetaData> inputs_metadata;
-            confidence_threshold, iou_threshold, num_of_classes, should_dequantize, max_boxes, should_sigmoid, one_class_per_bbox);
+        std::map<std::string, net_flow::BufferMetaData> outputs_metadata;
+
+        net_flow::NmsPostProcessConfig nms_post_process_config{};
+        nms_post_process_config.nms_score_th = confidence_threshold;
+        nms_post_process_config.nms_iou_th = iou_threshold;
+        nms_post_process_config.max_proposals_per_class = max_boxes;
+        nms_post_process_config.classes = num_of_classes;
+        nms_post_process_config.background_removal = false;
+        nms_post_process_config.background_removal_index = 0;
+        nms_post_process_config.cross_classes = cross_classes;
+        net_flow::YoloPostProcessConfig yolo_post_process_config{};
+        yolo_post_process_config.image_height = image_height;
+        yolo_post_process_config.image_width = image_width;
+        // Each layer anchors vector is structured as {w,h} pairs.
+        for (size_t i = 0; i < anchors.size(); ++i) {
+            auto name = std::to_string(i);
+            yolo_post_process_config.anchors.insert({name, anchors[i]});
+            BufferMetaData input_metadata = {
+                shapes[i],
+                shapes[i],
+                formats[i],
+                quant_infos[i]
+            };
+            inputs_metadata.insert({name, input_metadata});
+        }
+        auto op = YOLOv5PostProcessOp::create(inputs_metadata, outputs_metadata, nms_post_process_config, yolo_post_process_config);
         VALIDATE_EXPECTED(op);
 
-        return YOLOv5PostProcessingOpWrapper(op.release(), num_of_classes, max_boxes);
+        return YOLOv5PostProcessOpWrapper(op.release(), num_of_classes, max_boxes);
     }
 
     static void add_to_python_module(py::module &m)
     {
-        py::class_<YOLOv5PostProcessingOpWrapper>(m, "YOLOv5PostProcessingOp")
+        py::class_<YOLOv5PostProcessOpWrapper>(m, "YOLOv5PostProcessOp")
-        .def("create", &YOLOv5PostProcessingOpWrapper::create)
+        .def("create", &YOLOv5PostProcessOpWrapper::create)
-        .def("execute",[](YOLOv5PostProcessingOpWrapper &self, const std::vector<py::array> &tensors)
+        .def("execute",[](YOLOv5PostProcessOpWrapper &self, const std::vector<py::array> &tensors)
         {
-            std::vector<MemoryView> data_views;
+            std::map<std::string, MemoryView> data_views;
-            data_views.reserve(tensors.size());
+            for (size_t i = 0; i < tensors.size(); ++i) {
-            for (auto &tensor : tensors) {
+                data_views.insert({std::to_string(i),
-                data_views.push_back(MemoryView(const_cast<void*>(reinterpret_cast<const void*>(tensor.data())), tensor.nbytes()));
+                    MemoryView(const_cast<void*>(reinterpret_cast<const void*>(tensors[i].data())), tensors[i].nbytes())});
             }
 
             hailo_nms_info_t nms_info = {
@@ -64,7 +92,9 @@ public:
 
             auto buffer = Buffer::create(HailoRTCommon::get_nms_host_frame_size(nms_info, output_format), 0);
             VALIDATE_STATUS(buffer.status());
-            auto status = self.m_post_processing_op.execute<float32_t>(data_views, MemoryView(buffer.value().data(), buffer.value().size()));
+            std::map<std::string, MemoryView> outputs;
+            outputs.insert({"", MemoryView(buffer.value().data(), buffer.value().size())});
+            auto status = self.m_post_processing_op->execute(data_views, outputs);
             VALIDATE_STATUS(status);
 
             // Note: The ownership of the buffer is transferred to Python wrapped as a py::array.
@@ -80,19 +110,19 @@ public:
     }
 
 private:
-    YOLOv5PostProcessingOpWrapper(YOLOv5PostProcessingOp &&post_processing_op, uint32_t num_of_classes, uint32_t max_bboxes)
+    YOLOv5PostProcessOpWrapper(std::shared_ptr<Op> post_processing_op, uint32_t num_of_classes, uint32_t max_bboxes)
         : m_post_processing_op(post_processing_op),
           m_num_of_classes(num_of_classes),
           m_max_boxes(max_bboxes) {}
 
-    YOLOv5PostProcessingOp m_post_processing_op;
+    std::shared_ptr<Op> m_post_processing_op;
     uint32_t m_num_of_classes = 0;
     uint32_t m_max_boxes = 0;
 };
 
 void NetFlow_api_initialize_python_module(py::module &m)
 {
-    YOLOv5PostProcessingOpWrapper::add_to_python_module(m);
+    YOLOv5PostProcessOpWrapper::add_to_python_module(m);
 }
 
 

hailort/libhailort/bindings/python/src/pyhailort.cpp

@@ -8,7 +8,8 @@
 #include <exception>
 using namespace std;
 
-#include "hailo/hailort.hpp"
+#include "hailo/hailort.h"
+#include "hailo/hailort_defaults.hpp"
 
 #include "hef_api.hpp"
 #include "vstream_api.hpp"
@@ -23,7 +24,6 @@ using namespace std;
 #include "bindings_common.hpp"
 
 #include "sensor_config_exports.h"
-#include "hailort_defaults.hpp"
 #if defined(__GNUC__)
 #include "common/os/posix/traffic_control.hpp"
 #endif
@@ -44,6 +44,7 @@ bool hailo_format_equals(hailo_format_t &first, hailo_format_t &second){
         (first.order == second.order) &&
         (first.flags == second.flags));
 }
+
 class UdpScan {
     public:
         UdpScan() = default;
@@ -231,7 +232,7 @@ PYBIND11_MODULE(_pyhailort, m) {
         .value("HAILO8_A0", HAILO_ARCH_HAILO8_A0)
         .value("HAILO8", HAILO_ARCH_HAILO8)
         .value("HAILO8L", HAILO_ARCH_HAILO8L)
-        .value("MERCURY_CA", HAILO_ARCH_MERCURY_CA)
+        .value("HAILO15", HAILO_ARCH_HAILO15)
     ;
 
     /* TODO: SDK-15648 */
@@ -539,6 +540,9 @@ PYBIND11_MODULE(_pyhailort, m) {
         .value("YYUV", HAILO_FORMAT_ORDER_HAILO_YYUV)
         .value("NV21", HAILO_FORMAT_ORDER_NV21)
         .value("YYVU", HAILO_FORMAT_ORDER_HAILO_YYVU)
+        .value("RGB4", HAILO_FORMAT_ORDER_RGB4)
+        .value("I420", HAILO_FORMAT_ORDER_I420)
+        .value("YYYYUV", HAILO_FORMAT_ORDER_HAILO_YYYYUV)
         ;
 
     py::enum_<hailo_format_flags_t>(m, "FormatFlags", py::arithmetic())
@@ -727,11 +731,11 @@ PYBIND11_MODULE(_pyhailort, m) {
         .def(py::init<>())
         ;
 
-    py::class_<hailo_core_input_stream_params_t>(m, "CoreInputStreamParams")
+    py::class_<hailo_integrated_input_stream_params_t>(m, "IntegratedInputStreamParams")
         .def(py::init<>())
         ;
 
-    py::class_<hailo_core_output_stream_params_t>(m, "CoreOutputStreamParams")
+    py::class_<hailo_integrated_output_stream_params_t>(m, "IntegratedOutputStreamParams")
         .def(py::init<>())
         ;
 
@@ -746,7 +750,7 @@ PYBIND11_MODULE(_pyhailort, m) {
 
     py::enum_<hailo_stream_interface_t>(m, "StreamInterface")
         .value("PCIe", HAILO_STREAM_INTERFACE_PCIE)
-        .value("CORE", HAILO_STREAM_INTERFACE_CORE)
+        .value("INTEGRATED", HAILO_STREAM_INTERFACE_INTEGRATED)
         .value("ETH", HAILO_STREAM_INTERFACE_ETH)
         .value("MIPI", HAILO_STREAM_INTERFACE_MIPI)
         ;
@@ -772,12 +776,18 @@ PYBIND11_MODULE(_pyhailort, m) {
     py::class_<hailo_activate_network_group_params_t>(m, "ActivateNetworkGroupParams")
         .def(py::init<>())
         .def_static("default", []() {
-            return HailoRTDefaults::get_network_group_params();
+            return HailoRTDefaults::get_active_network_group_params();
         });
         ;
 
+    py::enum_<hailo_scheduling_algorithm_t>(m, "SchedulingAlgorithm")
+        .value("NONE", HAILO_SCHEDULING_ALGORITHM_NONE)
+        .value("ROUND_ROBIN", HAILO_SCHEDULING_ALGORITHM_ROUND_ROBIN)
+    ;
+
     py::class_<VDeviceParamsWrapper>(m, "VDeviceParams")
         .def(py::init<>())
+        // Add device_ids
         .def_property("device_count",
             [](const VDeviceParamsWrapper& params) -> uint32_t {
                 return params.orig_params.device_count;
@@ -786,6 +796,14 @@ PYBIND11_MODULE(_pyhailort, m) {
                 params.orig_params.device_count = device_count;
             }        
         )
+        .def_property("scheduling_algorithm",
+            [](const VDeviceParamsWrapper& params) -> uint32_t {
+                return params.orig_params.scheduling_algorithm;
+            },
+            [](VDeviceParamsWrapper& params, hailo_scheduling_algorithm_t scheduling_algorithm) {
+                params.orig_params.scheduling_algorithm = scheduling_algorithm;
+            }        
+        )
         .def_property("group_id",
             [](const VDeviceParamsWrapper& params) -> py::str {
                 return std::string(params.orig_params.group_id);
@@ -795,6 +813,14 @@ PYBIND11_MODULE(_pyhailort, m) {
                 params.orig_params.group_id = params.group_id_str.c_str();
             }
         )
+        .def_property("multi_process_service",
+            [](const VDeviceParamsWrapper& params) -> uint32_t {
+                return params.orig_params.multi_process_service;
+            },
+            [](VDeviceParamsWrapper& params, bool multi_process_service) {
+                params.orig_params.multi_process_service = multi_process_service;
+            }
+        )
         .def_static("default", []() {
             auto orig_params = HailoRTDefaults::get_vdevice_params();
             orig_params.scheduling_algorithm = HAILO_SCHEDULING_ALGORITHM_NONE;
@@ -808,8 +834,8 @@ PYBIND11_MODULE(_pyhailort, m) {
         .def_readonly("direction", &hailo_stream_parameters_t::direction)
         STREAM_PARAMETERS_UNION_PROPERTY(pcie_input_params, hailo_pcie_input_stream_params_t,
             HAILO_STREAM_INTERFACE_PCIE, HAILO_H2D_STREAM)
-        STREAM_PARAMETERS_UNION_PROPERTY(core_input_params, hailo_core_input_stream_params_t,
+        STREAM_PARAMETERS_UNION_PROPERTY(integrated_input_params, hailo_integrated_input_stream_params_t,
-            HAILO_STREAM_INTERFACE_CORE, HAILO_H2D_STREAM)
+            HAILO_STREAM_INTERFACE_INTEGRATED, HAILO_H2D_STREAM)
         STREAM_PARAMETERS_UNION_PROPERTY(eth_input_params, hailo_eth_input_stream_params_t,
             HAILO_STREAM_INTERFACE_ETH, HAILO_H2D_STREAM)
         STREAM_PARAMETERS_UNION_PROPERTY(mipi_input_params, hailo_mipi_input_stream_params_t,
@@ -818,8 +844,8 @@ PYBIND11_MODULE(_pyhailort, m) {
             HAILO_STREAM_INTERFACE_PCIE, HAILO_D2H_STREAM)
         STREAM_PARAMETERS_UNION_PROPERTY(eth_output_params, hailo_eth_output_stream_params_t,
             HAILO_STREAM_INTERFACE_ETH, HAILO_D2H_STREAM)
-        STREAM_PARAMETERS_UNION_PROPERTY(core_output_params, hailo_core_output_stream_params_t,
+        STREAM_PARAMETERS_UNION_PROPERTY(integrated_output_params, hailo_integrated_output_stream_params_t,
-            HAILO_STREAM_INTERFACE_CORE, HAILO_D2H_STREAM)
+            HAILO_STREAM_INTERFACE_INTEGRATED, HAILO_D2H_STREAM)
         ;
 
     py::class_<hailo_network_parameters_t>(m, "NetworkParameters")
@@ -1058,6 +1084,7 @@ PYBIND11_MODULE(_pyhailort, m) {
         .def_readonly("name", &hailo_stream_info_t::name)
         .def_readonly("sys_index", &hailo_stream_info_t::index)
         .def_readonly("data_bytes", &hailo_stream_info_t::hw_data_bytes)
+        .def_readonly("quant_info", &hailo_stream_info_t::quant_info)
         .def("__repr__", [](const hailo_stream_info_t &self) {
             return std::string("StreamInfo(\"") + std::string(self.name) + std::string("\")");
         })

hailort/libhailort/bindings/python/src/quantization_api.cpp

@@ -7,6 +7,8 @@
  * @brief Quantization python bindings functions
  **/
 
+#include "hailo/quantization.hpp"
+
 #include "quantization_api.hpp"
 #include "bindings_common.hpp"
 

hailort/libhailort/bindings/python/src/quantization_api.hpp

@@ -10,12 +10,14 @@
 #ifndef _HAILO_QUANTIZATION_API_HPP_
 #define _HAILO_QUANTIZATION_API_HPP_
 
-#include "hailo/hailort.hpp"
+#include "hailo/hailort.h"
+
 #include "utils.hpp"
 
 #include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
 
+
 namespace hailort
 {
 

hailort/libhailort/bindings/python/src/utils.hpp

@@ -40,10 +40,10 @@ class HailoRTStatusException : public HailoRTException {
         }                                                                           \
     } while (0)
 
-#define VALIDATE_NOT_NULL(__ptr)                                                    \
+#define VALIDATE_NOT_NULL(__ptr, __status)                                          \
     do {                                                                            \
         if (nullptr == (__ptr)) {                                                   \
-            throw HailoRTStatusException(std::to_string(HAILO_INVALID_ARGUMENT));   \
+            throw HailoRTStatusException(std::to_string(__status));                 \
         }                                                                           \
     } while (0)
 

hailort/libhailort/bindings/python/src/vdevice_api.hpp

@@ -14,9 +14,15 @@
 
 #include "hailo/hef.hpp"
 #include "hailo/vdevice.hpp"
+#include "hailo/hailort_common.hpp"
+
+#include "common/logger_macros.hpp"
+
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+#include "service/rpc_client_utils.hpp"
+#endif // HAILO_SUPPORT_MULTI_PROCESS
 
 #include "utils.hpp"
-#include "common/logger_macros.hpp"
 
 #include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
@@ -27,6 +33,7 @@
 
 #include <string>
 
+
 namespace hailort
 {
 
@@ -47,6 +54,19 @@ public:
         return VDeviceWrapper(params.orig_params);
     }
 
+    static VDeviceWrapper create(const VDeviceParamsWrapper &params, const std::vector<std::string> &device_ids)
+    {
+        if (params.orig_params.device_ids != nullptr && (!device_ids.empty())) {
+            LOGGER__ERROR("VDevice device_ids can be set in params or device_ids argument. Both parameters were passed to the c'tor");
+            throw HailoRTStatusException(std::to_string(HAILO_INVALID_OPERATION));
+        }
+        auto modified_params = params;
+        auto device_ids_vector = HailoRTCommon::to_device_ids_vector(device_ids);
+        VALIDATE_EXPECTED(device_ids_vector);
+        modified_params.orig_params.device_ids = device_ids_vector->data();
+        return VDeviceWrapper(modified_params.orig_params);
+    }
+
     static VDeviceWrapper create_from_ids(const std::vector<std::string> &device_ids)
     {
         auto device_ids_vector = HailoRTCommon::to_device_ids_vector(device_ids);
@@ -87,8 +107,10 @@ public:
         VALIDATE_EXPECTED(network_groups);
 
         py::list results;
+        m_net_groups.reserve(m_net_groups.size() + network_groups->size());
         for (const auto &network_group : network_groups.value()) {
             results.append(network_group.get());
+            m_net_groups.emplace_back(network_group);
         }
 
         return results;
@@ -96,11 +118,43 @@ public:
 
     void release()
     {
+        m_net_groups.clear();
         m_vdevice.reset();
     }
 
+    void before_fork()
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        if (m_vdevice != nullptr) {
+            auto status = m_vdevice->before_fork();
+            VALIDATE_STATUS(status);
+        }
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+
+    void after_fork_in_parent()
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        if (m_vdevice != nullptr) {
+            auto status = m_vdevice->after_fork_in_parent();
+            VALIDATE_STATUS(status);
+        }
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+
+    void after_fork_in_child()
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        if (m_vdevice != nullptr) {
+            auto status = m_vdevice->after_fork_in_child();
+            VALIDATE_STATUS(status);
+        }
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+
 private:
     std::unique_ptr<VDevice> m_vdevice;
+    ConfiguredNetworkGroupVector m_net_groups;
 };
 
 void VDevice_api_initialize_python_module(py::module &m)
@@ -108,10 +162,14 @@ void VDevice_api_initialize_python_module(py::module &m)
     py::class_<VDeviceWrapper>(m, "VDevice")
         .def("create", py::overload_cast<const hailo_vdevice_params_t&>(&VDeviceWrapper::create))
         .def("create", py::overload_cast<const VDeviceParamsWrapper&>(&VDeviceWrapper::create))
+        .def("create", py::overload_cast<const VDeviceParamsWrapper&, const std::vector<std::string>&>(&VDeviceWrapper::create))
         .def("create_from_ids", &VDeviceWrapper::create_from_ids)
         .def("get_physical_devices_ids", &VDeviceWrapper::get_physical_devices_ids)
         .def("configure", &VDeviceWrapper::configure)
         .def("release", &VDeviceWrapper::release)
+        .def("before_fork", &VDeviceWrapper::before_fork)
+        .def("after_fork_in_parent", &VDeviceWrapper::after_fork_in_parent)
+        .def("after_fork_in_child", &VDeviceWrapper::after_fork_in_child)
         ;
 }
 

hailort/libhailort/bindings/python/src/vstream_api.cpp

@@ -27,6 +27,36 @@ void InputVStreamWrapper::add_to_python_module(py::module &m)
             MemoryView(const_cast<void*>(reinterpret_cast<const void*>(data.data())), data.nbytes()));
         VALIDATE_STATUS(status);
     })
+    .def("before_fork", [](InputVStream &self)
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        auto status = self.before_fork();
+        VALIDATE_STATUS(status);
+#else
+        (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+    )
+    .def("after_fork_in_parent", [](InputVStream &self)
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        auto status = self.after_fork_in_parent();
+        VALIDATE_STATUS(status);
+#else
+        (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+    )
+    .def("after_fork_in_child", [](InputVStream &self)
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        auto status = self.after_fork_in_child();
+        VALIDATE_STATUS(status);
+#else
+        (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+    )
     .def("flush", [](InputVStream &self)
     {
         hailo_status status = self.flush();
@@ -100,6 +130,36 @@ void InputVStreamsWrapper::clear()
     VALIDATE_STATUS(status);
 }
 
+void InputVStreamsWrapper::before_fork()
+{
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+    for (auto &input_vstream : m_input_vstreams) {
+        auto status = input_vstream.second->before_fork();
+        VALIDATE_STATUS(status);
+    }
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+}
+
+void InputVStreamsWrapper::after_fork_in_parent()
+{
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+    for (auto &input_vstream : m_input_vstreams) {
+        auto status = input_vstream.second->after_fork_in_parent();
+        VALIDATE_STATUS(status);
+    }
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+}
+
+void InputVStreamsWrapper::after_fork_in_child()
+{
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+    for (auto &input_vstream : m_input_vstreams) {
+        auto status = input_vstream.second->after_fork_in_child();
+        VALIDATE_STATUS(status);
+    }
+#endif // HAILO_SUPPORT_MULTI_PROCESS    
+}
+
 void InputVStreamsWrapper::add_to_python_module(py::module &m)
 {
     py::class_<InputVStreamsWrapper>(m, "InputVStreams")
@@ -109,6 +169,9 @@ void InputVStreamsWrapper::add_to_python_module(py::module &m)
     .def("clear", &InputVStreamsWrapper::clear)
     .def("__enter__", &InputVStreamsWrapper::enter, py::return_value_policy::reference)
     .def("__exit__",  [&](InputVStreamsWrapper &self, py::args) { self.exit(); })
+    .def("before_fork", &InputVStreamsWrapper::before_fork)
+    .def("after_fork_in_parent",  &InputVStreamsWrapper::after_fork_in_parent)
+    .def("after_fork_in_child", &InputVStreamsWrapper::after_fork_in_child)
     ;
 }
 
@@ -151,6 +214,36 @@ void OutputVStreamWrapper::add_to_python_module(py::module &m)
         return py::array(get_dtype(self), get_shape(self), unmanaged_addr,
             py::capsule(unmanaged_addr, [](void *p) { delete reinterpret_cast<uint8_t*>(p); }));
     })
+    .def("before_fork", [](OutputVStream &self)
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        auto status = self.before_fork();
+        VALIDATE_STATUS(status);
+#else
+        (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+    )
+    .def("after_fork_in_parent", [](OutputVStream &self)
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        auto status = self.after_fork_in_parent();
+        VALIDATE_STATUS(status);
+#else
+        (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+    )
+    .def("after_fork_in_child", [](OutputVStream &self)
+    {
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+        auto status = self.after_fork_in_child();
+        VALIDATE_STATUS(status);
+#else
+        (void)self;
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+    }
+    )
     .def_property_readonly("info", [](OutputVStream &self)
     {
         return self.get_info();
@@ -213,6 +306,36 @@ void OutputVStreamsWrapper::clear()
     VALIDATE_STATUS(status);
 }
 
+void OutputVStreamsWrapper::before_fork()
+{
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+    for (auto &output_vstream : m_output_vstreams) {
+        auto status = output_vstream.second->before_fork();
+        VALIDATE_STATUS(status);
+    }
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+}
+
+void OutputVStreamsWrapper::after_fork_in_parent()
+{
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+    for (auto &output_vstream : m_output_vstreams) {
+        auto status = output_vstream.second->after_fork_in_parent();
+        VALIDATE_STATUS(status);
+    }
+#endif // HAILO_SUPPORT_MULTI_PROCESS
+}
+
+void OutputVStreamsWrapper::after_fork_in_child()
+{
+#ifdef HAILO_SUPPORT_MULTI_PROCESS
+    for (auto &output_vstream : m_output_vstreams) {
+        auto status = output_vstream.second->after_fork_in_child();
+        VALIDATE_STATUS(status);
+    }
+#endif // HAILO_SUPPORT_MULTI_PROCESS    
+}
+
 void OutputVStreamsWrapper::add_to_python_module(py::module &m)
 {
     py::class_<OutputVStreamsWrapper>(m, "OutputVStreams")
@@ -222,6 +345,9 @@ void OutputVStreamsWrapper::add_to_python_module(py::module &m)
     .def("clear", &OutputVStreamsWrapper::clear)
     .def("__enter__", &OutputVStreamsWrapper::enter, py::return_value_policy::reference)
     .def("__exit__",  [&](OutputVStreamsWrapper &self, py::args) { self.exit(); })
+    .def("before_fork", &OutputVStreamsWrapper::before_fork)
+    .def("after_fork_in_parent",  &OutputVStreamsWrapper::after_fork_in_parent)
+    .def("after_fork_in_child", &OutputVStreamsWrapper::after_fork_in_child)
     ;
 }
 

hailort/libhailort/bindings/python/src/vstream_api.hpp

@@ -41,6 +41,9 @@ public:
     std::shared_ptr<InputVStream> get_input_by_name(const std::string &name);
     py::dict get_all_inputs();
     void clear();
+    void before_fork();
+    void after_fork_in_parent();
+    void after_fork_in_child();
     static void add_to_python_module(py::module &m);
 
 private:
@@ -67,6 +70,9 @@ public:
     void exit();
     py::dict get_all_outputs();
     void clear();
+    void before_fork();
+    void after_fork_in_parent();
+    void after_fork_in_child();
     static void add_to_python_module(py::module &m);
 
 private:

hailort/libhailort/examples/README.md

@@ -26,6 +26,7 @@ The following examples are provided, demonstrating the HailoRT API:
     - this example uses udp device.
   - `raw_streams_example` - Basic inference of a shortcut network using raw stream api.
     - The data is transformed before sent and after received in the same thread sending/receiving using the transformation api.
+  - `notification_callback_example` - Demonstrates how to work with notification callbacks.
 
 - C++ examples:
   - `vstreams_example` - Basic inference of a shortcut network, same as `vstreams_example` C example, uses HailoRT C++ api.
@@ -39,7 +40,7 @@ The following examples are provided, demonstrating the HailoRT API:
   - `raw_streams_example` - Basic inference of a shortcut network, same as `raw_streams_example` C example, uses HailoRT C++ api.
   - `multi_process_example` - Demonstrates how to work with HailoRT as a service and using the HailoRT Model Scheduler for network groups switching.
   Using the script `multi_process_example.sh` one can specify the number of processes to run each hef, see `multi_process_example.sh -h` for more information.
-
+  - `notification_callback_example` - Demonstrates how to work with notification callbacks, same as `notification_callback_example` C example.
 ## Compiling with CMake
 Examples are configured and compiled using the following commands:
 ```sh

hailort/libhailort/examples/c/CMakeLists.txt

@@ -9,6 +9,7 @@ add_subdirectory(switch_network_groups_example)
 add_subdirectory(switch_network_groups_manually_example)
 add_subdirectory(multi_device_example)
 add_subdirectory(power_measurement_example)
+add_subdirectory(notification_callback_example)
 
 add_custom_target(c_hailort_examples)
 add_dependencies(c_hailort_examples
@@ -20,4 +21,5 @@ add_dependencies(c_hailort_examples
     c_switch_network_groups_example
     c_switch_network_groups_manually_example
     c_multi_device_example
-    c_power_measurement_example)
+    c_power_measurement_example
+    c_notification_callback_example)

hailort/libhailort/examples/c/data_quantization_example/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0.0)
 find_package(Threads REQUIRED)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 
-find_package(HailoRT 4.12.1 EXACT REQUIRED)
+find_package(HailoRT 4.13.0 EXACT REQUIRED)
 
 SET_SOURCE_FILES_PROPERTIES(data_quantization_example.c PROPERTIES LANGUAGE C)