dao_graph_feature_arc_worker.h

Go to the documentation of this file.

/* SPDX-License-Identifier: Marvell-MIT
 * Copyright (c) 2024 Marvell.
 */
 
#ifndef _DAO_GRAPH_FEATURE_ARC_WORKER_H_
#define _DAO_GRAPH_FEATURE_ARC_WORKER_H_
 
#include <stddef.h>
#include <stdalign.h>
#include <dao_graph_feature_arc.h>
#include <rte_bitops.h>
 
#ifdef __cplusplus
extern "C" {
#endif
 
struct __rte_cache_aligned dao_graph_feature_node_list {
    STAILQ_ENTRY(dao_graph_feature_node_list) next_feature;
 
    struct rte_node_register *feature_node;
 
    int32_t ref_count;
 
    void *feature_arc;
 
    rte_edge_t edge_to_this_feature;
};
 
typedef struct dao_graph_feature_data {
    int64_t data;
 
    uint32_t feature_data_index;
 
    rte_edge_t edge_to_this_feature;
 
    rte_edge_t edge_to_next_feature;
 
    struct dao_graph_feature_node_list *node_info;
} dao_graph_feature_data_t;
 
struct __rte_cache_aligned dao_graph_feature {
    struct dao_graph_feature_node_list *node_info;
 
    void *feature_arc;
 
    uint32_t num_enabled_features;
 
    /* uint32_t reserved; */
 
    struct dao_graph_feature_data feature_data[DAO_GRAPH_FEATURE_MAX_PER_ARC];
};
 
struct __rte_cache_aligned dao_graph_feature_arc {
    STAILQ_HEAD(, dao_graph_feature_node_list) all_features;
 
    char feature_arc_name[DAO_GRAPH_FEATURE_ARC_NAMELEN];
 
    uint32_t feature_arc_index;
 
    void *feature_arc_main;
 
    struct rte_node_register *start_node;
 
    uint32_t max_features;
 
    uint32_t max_indexes;
 
    /* Fast path stuff*/
    alignas(RTE_CACHE_LINE_SIZE) RTE_MARKER cacheline1;
    int runtime_enabled_features;
 
    struct dao_graph_feature *features_by_index;
 
    uint64_t feature_bit_mask_by_index[];
};
 
typedef struct feature_arc_main {
    uint32_t num_feature_arcs;
 
    uint32_t max_feature_arcs;
 
    dao_graph_feature_arc_t feature_arcs[];
} dao_graph_feature_arc_main_t;
 
#define dao_graph_feature_arc_get(dfl) ((struct dao_graph_feature_arc *)dfl)
 
extern dao_graph_feature_arc_main_t *__feature_arc_main;
 
static inline int
__bsf64_safe(uint64_t v, uint8_t *pos)
{
    if (v == 0)
        return 0;
 
    *pos = (uint8_t)rte_ctz64(v);
    return 1;
}
 
static inline struct dao_graph_feature_data *
dao_graph_feature_data_get(struct dao_graph_feature *df, uint8_t feature_index)
{
    return (df->feature_data + feature_index);
}
 
static inline struct dao_graph_feature *
dao_graph_feature_get(struct dao_graph_feature_arc *dfl, uint32_t index)
{
    return (dfl->features_by_index + index);
}
 
static inline int
dao_graph_feature_arc_has_first_feature(struct dao_graph_feature_arc *dfl,
                    uint32_t index, dao_graph_feature_t *feature)
{
    return __bsf64_safe(dfl->feature_bit_mask_by_index[index], feature);
}
 
static inline int
dao_graph_feature_arc_has_next_feature(struct dao_graph_feature_arc *dfl,
                       uint32_t index, dao_graph_feature_t *feature)
{
    uint64_t bitmask;
 
#ifdef DAO_GRAPH_FEATURE_ARC_DEBUG
    struct dao_graph_feature *df = dao_graph_feature_get(dfl, index);
    struct dao_graph_feature_data *dfd = NULL;
 
    dfd = dao_graph_feature_data_get(df, *feature);
    if (unlikely(dfd->feature_data_index != *feature))
        return 0;
#endif
 
    /* Create bitmask where current feature is cleared to get next feature
     * bit set
     */
    bitmask = UINT64_MAX << (*feature + 1);
    bitmask = dfl->feature_bit_mask_by_index[index] & bitmask;
 
    return __bsf64_safe(bitmask, feature);
}
 
static inline void
dao_graph_feature_arc_prefetch(struct dao_graph_feature_arc *dfl, uint32_t index)
{
    rte_prefetch0((uint8_t *)dfl +
              (offsetof(struct dao_graph_feature_arc, feature_bit_mask_by_index) +
              (index * sizeof(uint64_t *))));
}
 
static inline int
dao_graph_feature_arc_num_enabled_features(struct dao_graph_feature_arc *arc)
{
    return arc->runtime_enabled_features;
}
 
static inline int
dao_graph_feature_arc_has_feature(struct dao_graph_feature_arc *dfl, uint32_t index,
                  dao_graph_feature_t *feature)
{
#ifdef DAO_GRAPH_FEATURE_ARC_DEBUG
    if (unlikely(dfl->max_indexes < index))
        return 0;
 
    if (unlikely(!feature))
        return 0;
#endif
    if (likely(!dfl->feature_bit_mask_by_index[index]))
        return 0;
 
    /* Look for first feature */
    if (*feature == DAO_GRAPH_FEATURE_INVALID_VALUE)
        return dao_graph_feature_arc_has_first_feature(dfl, index, feature);
    else
        return dao_graph_feature_arc_has_next_feature(dfl, index, feature);
}
 
static inline void
__dao_graph_prefetch_data_prefetch(struct dao_graph_feature_arc *dfl, int index,
                   dao_graph_feature_t feature)
{
    struct dao_graph_feature *df = dao_graph_feature_get(dfl, index);
 
    rte_prefetch0((void *)dao_graph_feature_data_get(df, feature));
}
 
static inline void
dao_graph_feature_data_prefetch(struct dao_graph_feature_arc *dfl, uint32_t index,
                dao_graph_feature_t feature)
{
#ifdef DAO_GRAPH_FEATURE_ARC_DEBUG
    if (unlikely(index >= dfl->max_indexes))
        return;
 
    if (unlikely(feature >= dao_graph_feature_cast(dfl->max_features)))
        return;
#endif
    if (feature != DAO_GRAPH_FEATURE_INVALID_VALUE)
        __dao_graph_prefetch_data_prefetch(dfl, index, feature);
}
 
static inline int
dao_graph_feature_arc_first_feature_data_get(struct dao_graph_feature_arc *dfl,
                         dao_graph_feature_t feature,
                         uint32_t index, rte_edge_t *edge,
                         int64_t *data)
{
    struct dao_graph_feature *df = dao_graph_feature_get(dfl, index);
    struct dao_graph_feature_data *dfd = NULL;
 
    dfd = dao_graph_feature_data_get(df, feature);
 
#ifdef DAO_GRAPH_FEATURE_ARC_DEBUG
    if (unlikely(dfd->feature_data_index != feature))
        return -1;
 
    if (unlikely(!edge && !data))
        return -1;
#endif
 
    *edge = dfd->edge_to_this_feature;
    *data = dfd->data;
 
    return 0;
}
 
static inline int
dao_graph_feature_arc_next_feature_data_get(struct dao_graph_feature_arc *dfl,
                        dao_graph_feature_t feature,
                        uint32_t index, rte_edge_t *edge,
                        int64_t *data)
{
    struct dao_graph_feature *df = dao_graph_feature_get(dfl, index);
    struct dao_graph_feature_data *dfd = NULL;
 
    dfd = dao_graph_feature_data_get(df, feature);
 
#ifdef DAO_GRAPH_FEATURE_ARC_DEBUG
    if (unlikely(dfd->feature_data_index != feature))
        return -1;
 
    if (unlikely(!edge && !data))
        return -1;
#endif
 
    *edge = dfd->edge_to_next_feature;
    *data = dfd->data;
 
    return 0;
}
 
static inline int
dao_graph_feature_arc_feature_data_get(struct dao_graph_feature_arc *dfl,
                       dao_graph_feature_t feature,
                       uint32_t index, rte_edge_t *edge,
                       int64_t *data)
{
    if (feature == DAO_GRAPH_FEATURE_INVALID_VALUE)
        return dao_graph_feature_arc_first_feature_data_get(dfl, feature, index, edge,
                                    data);
    else
        return dao_graph_feature_arc_next_feature_data_get(dfl, feature, index, edge, data);
}
 
#ifdef __cplusplus
}
#endif
#endif