dao_workers.h

Go to the documentation of this file.

/* SPDX-License-Identifier: Marvell-MIT
 * Copyright (c) 2023 Marvell.
 */
 
#ifndef _DAO_LIB_WORKERS_H_
#define _DAO_LIB_WORKERS_H_
 
#include <errno.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stddef.h>
#include <signal.h>
#include <assert.h>
 
#include <rte_common.h>
#include <rte_compat.h>
#include <rte_debug.h>
#include <rte_lcore.h>
#include <rte_bitops.h>
#include <rte_atomic.h>
#include <rte_pause.h>
 
#include <dao_log.h>
#include <dao_util.h>
 
#ifdef __cplusplus
extern "C" {
#endif
 
#define DAO_WORKER_INVALID_INDEX UINT32_C(~0)
 
#define DAO_WORKERS_LOG dao_dbg
 
typedef struct dao_worker {
    unsigned int dpdk_lcore_id;
 
    int dpdk_cpu_id;
 
    int dpdk_core_index;
 
    uint32_t worker_index;
 
    uint32_t core_index;
 
    int dpdk_numa_id;
 
    /* is this worker dpdk_main_core where rte_eal_init*/
    int is_main;
 
    /* back pointer to dao_workers_main */
    void *dao_workers;
 
    /* Size of app_private[] on worker */
    size_t app_private_size;
 
    RTE_MARKER c1 __rte_cache_aligned;
    uint8_t app_private[];
} dao_worker_t __rte_cache_aligned;
 
typedef struct dao_workers_main {
    RTE_MARKER c0 __rte_cache_aligned;
 
    uint16_t num_cores;
 
    uint16_t num_workers;
 
    uint64_t core_mask;
 
    uint32_t control_core_index;
 
    size_t per_worker_sz;
 
    size_t workers_main_sz;
 
    int barrier_recursion_level;
 
    RTE_MARKER c1  __rte_cache_aligned;
    uint64_t parked_at_barrier;
 
    RTE_MARKER c2  __rte_cache_aligned;
    uint64_t barrier_count;
 
    RTE_MARKER c3 __rte_cache_aligned;
    dao_worker_t workers[];
} dao_workers_main_t;
 
extern dao_workers_main_t *__dao_workers;
 
/* Static inline functions */
 
static inline dao_worker_t *
dao_workers_worker_get(dao_workers_main_t *dwm, uint8_t worker_index)
{
    RTE_VERIFY(dwm);
    RTE_VERIFY(worker_index <= dwm->num_workers);
 
    return ((dao_worker_t *)((uint8_t *)dwm->workers + (worker_index * dwm->per_worker_sz)));
}
 
static inline int
dao_workers_app_data_get(dao_worker_t *wrkr, void **app_data, size_t *size)
{
    if (likely(wrkr)) {
        if (app_data)
            *app_data = wrkr->app_private;
 
        if (size)
            *size = wrkr->app_private_size;
 
        return 0;
    }
    return -1;
}
 
static inline int
dao_workers_core_index_get(dao_worker_t *wrkr)
{
    return wrkr->core_index;
}
 
static inline int
dao_workers_worker_index_get(dao_worker_t *wrkr)
{
    return wrkr->worker_index;
}
 
static inline int
dao_workers_numa_get(dao_worker_t *wrkr)
{
    return wrkr->dpdk_numa_id;
}
 
static inline int dao_workers_is_control_worker(dao_worker_t *worker)
{
    return worker->is_main;
}
 
static inline dao_worker_t *
dao_workers_control_worker_get(dao_workers_main_t *dwm)
{
    if (likely(dwm && (dwm->control_core_index != DAO_WORKER_INVALID_INDEX)))
        return (dwm->workers + dwm->control_core_index);
    return NULL;
}
 
static inline dao_workers_main_t *dao_workers_get(void)
{
    return __dao_workers;
}
 
static inline int
__dao_workers_num_workers_get(dao_workers_main_t *dwm)
{
    return dwm->num_workers;
}
 
static inline int dao_workers_num_workers_get(void)
{
    return __dao_workers_num_workers_get(dao_workers_get());
}
 
static inline int dao_workers_num_cores_get(void)
{
    dao_workers_main_t *dwm = dao_workers_get();
 
    return dwm->num_cores;
}
 
static inline dao_worker_t *
dao_workers_self_worker_get(void)
{
    dao_workers_main_t *wm = dao_workers_get();
    dao_worker_t *wrkr = NULL;
    unsigned int lcore_id;
    uint16_t i;
 
    lcore_id = rte_lcore_id();
 
    for (i = 0; i < wm->num_cores; i++) {
        wrkr = dao_workers_worker_get(wm, i);
 
        /* wrkr is sane */
        assert(wrkr->core_index == i);
        assert(wm == wrkr->dao_workers);
 
        if (wrkr->dpdk_lcore_id != lcore_id)
            continue;
 
        if (LCORE_ID_ANY == lcore_id)
            continue;
 
        break;
    }
    if (i < wm->num_cores)
        return (dao_workers_worker_get(wm, i));
 
    /* We should not reach here */
    return NULL;
}
 
/* Function declaration */
 
int dao_workers_init(uint64_t core_mask, uint32_t control_core_index, size_t per_core_app_data_sz);
 
int dao_workers_fini(void);
int dao_workers_barrier_sync(dao_worker_t *worker);
 
int dao_workers_barrier_release(dao_worker_t *worker);
 
static inline void
dao_workers_barrier_check(dao_worker_t *worker)
{
    dao_workers_main_t *dwm = (dao_workers_main_t *)worker->dao_workers;
 
    if (unlikely(__atomic_load_n(&dwm->parked_at_barrier, __ATOMIC_ACQUIRE))) {
        DAO_WORKERS_LOG("Worker%d: going to barrier 0x%lx ",
                dao_workers_worker_index_get(worker),
                __atomic_load_n(&dwm->barrier_count, __ATOMIC_RELAXED));
 
        __atomic_add_fetch(&dwm->barrier_count, 1, __ATOMIC_RELEASE);
 
        /* Busy wait worker core until control-core releases barrier */
        while (__atomic_load_n(&dwm->parked_at_barrier, __ATOMIC_RELAXED))
            rte_pause();
 
        DAO_WORKERS_LOG("Worker: %d released from barrier",
                dao_workers_worker_index_get(worker));
        }
}
 
#ifdef __cplusplus
}
#endif
#endif