From 2e07f5dcb0ba9c5be6b59928f2c8c665e201db14 Mon Sep 17 00:00:00 2001 From: w00808059 Date: Wed, 17 Dec 2025 17:36:04 +0800 Subject: [PATCH] adapt_ubcore --- drivers/ub/urma/ubagg/ubagg_ioctl.c | 292 ++++++----- drivers/ub/urma/ubagg/ubagg_ioctl.h | 4 +- drivers/ub/urma/ubagg/ubagg_topo_info.c | 6 +- drivers/ub/urma/ubagg/ubagg_topo_info.h | 38 +- drivers/ub/urma/ubcore/ubcore_cmd.h | 2 +- .../ub/urma/ubcore/ubcore_connect_bonding.c | 32 +- drivers/ub/urma/ubcore/ubcore_genl_admin.c | 2 +- drivers/ub/urma/ubcore/ubcore_topo_info.c | 479 ++++++++++-------- drivers/ub/urma/ubcore/ubcore_topo_info.h | 42 +- drivers/ub/urma/ubcore/ubcore_uvs_cmd.c | 108 ++-- include/ub/urma/ubcore_types.h | 2 +- 11 files changed, 570 insertions(+), 437 deletions(-) diff --git a/drivers/ub/urma/ubagg/ubagg_ioctl.c b/drivers/ub/urma/ubagg/ubagg_ioctl.c index 8f7545306f78..5ca5ec30b6c2 100644 --- a/drivers/ub/urma/ubagg/ubagg_ioctl.c +++ b/drivers/ub/urma/ubagg/ubagg_ioctl.c @@ -79,7 +79,7 @@ void ubagg_dev_ref_put(struct ubagg_device *dev) struct ubagg_dev_name_eid_arr { char master_dev_name[UBAGG_MAX_DEV_NAME_LEN]; - char bonding_eid[EID_LEN]; + char aggr_eid[EID_LEN]; }; static struct ubagg_dev_name_eid_arr g_name_eid_arr[UBAGG_MAX_BONDING_DEV_NUM] = { 0 }; @@ -1230,6 +1230,13 @@ static int rmv_dev(struct ubagg_cmd_hdr *hdr) return 0; } +static bool is_aggr_dev_valid(struct ubagg_topo_aggr_dev *aggr_dev) +{ + struct ubagg_topo_aggr_dev empty_dev = {0}; + + return (memcmp(aggr_dev, &empty_dev, sizeof(struct ubagg_topo_aggr_dev)) == 0) ? false : true; +} + static bool is_eid_valid(const char *eid) { int i; @@ -1241,23 +1248,23 @@ static bool is_eid_valid(const char *eid) return false; } -static bool is_bonding_and_primary_eid_valid(struct ubagg_topo_map *topo_map) +static bool is_aggr_and_primary_eid_valid(struct ubagg_topo_map *topo_map) { - int i, j; + int node_id, dev_id, iodie_id; bool has_primary_eid = false; - for (i = 0; i < topo_map->node_num; i++) { - if (!is_eid_valid(topo_map->topo_infos[i].bonding_eid)) - return false; - has_primary_eid = false; - for (j = 0; j < IODIE_NUM; j++) { - if (is_eid_valid(topo_map->topo_infos[i] - .io_die_info[j] - .primary_eid)) - has_primary_eid = true; + for (node_id = 0; node_id < topo_map->node_num; node_id++) { + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (!is_eid_valid(topo_map->topo_infos[node_id].devs[dev_id].aggr_eid)) + return false; + has_primary_eid = false; + for (iodie_id = 0; iodie_id < IODIE_NUM; iodie_id++) { + if (is_eid_valid(topo_map->topo_infos[node_id].devs[dev_id].fe[iodie_id].primary_eid)) + has_primary_eid = true; + } + if (!has_primary_eid) + return false; } - if (!has_primary_eid) - return false; } return true; } @@ -1268,7 +1275,7 @@ static int find_cur_node_index(struct ubagg_topo_map *topo_map, int i; for (i = 0; i < topo_map->node_num; i++) { - if (topo_map->topo_infos[i].is_cur_node) { + if (topo_map->topo_infos[i].is_current) { *node_index = i; break; } @@ -1280,43 +1287,51 @@ static int find_cur_node_index(struct ubagg_topo_map *topo_map, return 0; } -static bool compare_eids(const char *eid1, const char *eid2) +static bool is_eid_match(const char *eid1, const char *eid2) { return memcmp(eid1, eid2, EID_LEN) == 0; } -static int update_peer_port_eid(struct ubagg_topo_info *new_topo_info, - struct ubagg_topo_info *old_topo_info) +static void update_dev_info(struct ubagg_topo_node *new_topo_info, + struct ubagg_topo_node *old_topo_info) { - int i, j; - char *new_peer_port_eid; - char *old_peer_port_eid; - - for (i = 0; i < IODIE_NUM; i++) { - for (j = 0; j < MAX_PORT_NUM; j++) { - if (!is_eid_valid( - new_topo_info->io_die_info[i].port_eid[j])) - continue; + int dev_id; - new_peer_port_eid = - new_topo_info->io_die_info[i].peer_port_eid[j]; - old_peer_port_eid = - old_topo_info->io_die_info[i].peer_port_eid[j]; - - if (!is_eid_valid(new_peer_port_eid)) - continue; - if (is_eid_valid(old_peer_port_eid) && - !compare_eids(new_peer_port_eid, - old_peer_port_eid)) { - ubagg_log_err( - "peer port eid is not same, new: " EID_FMT - ", old: " EID_FMT "\n", - EID_RAW_ARGS(new_peer_port_eid), - EID_RAW_ARGS(old_peer_port_eid)); - return -1; + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (is_aggr_dev_valid(&old_topo_info->devs[dev_id])) { + (void)memcpy(&old_topo_info->devs[dev_id], &new_topo_info->devs[dev_id], + sizeof(struct ubagg_topo_aggr_dev)); + } + // TODO: update eid + } +} + +static int update_link_info(struct ubagg_topo_node *new_topo_info, + struct ubagg_topo_node *old_topo_info) +{ + int iodie_id, port_id, old_remote_port_id; + + for (iodie_id = 0; iodie_id < IODIE_NUM; iodie_id++) { + for (port_id = 0; port_id < MAX_PORT_NUM; ++port_id) { + // add new connection if no connection exists + old_remote_port_id = old_topo_info->links[iodie_id][port_id].peer_port; + if (old_remote_port_id == UINT32_MAX) { + (void)memcpy(&old_topo_info->links[iodie_id][port_id], + &new_topo_info->links[iodie_id][port_id], sizeof(struct ubagg_topo_link)); + } else { + if (memcmp(&old_topo_info->links[iodie_id][port_id], &new_topo_info->links[iodie_id][port_id], + sizeof(struct ubagg_topo_link)) != 0) { + ubagg_log_err("link is not the same, new: peer_node[%u]/peer_iodie[%u]/peer_port[%u],\ + old: peer_node[%u]/peer_iodie[%u]/peer_port[%u]\n", + new_topo_info->links[iodie_id][port_id].peer_node, + new_topo_info->links[iodie_id][port_id].peer_iodie, + new_topo_info->links[iodie_id][port_id].peer_port, + old_topo_info->links[iodie_id][port_id].peer_node, + old_topo_info->links[iodie_id][port_id].peer_iodie, + old_topo_info->links[iodie_id][port_id].peer_port); + return -EINVAL; + } } - (void)memcpy(old_peer_port_eid, new_peer_port_eid, - EID_LEN); } } return 0; @@ -1325,8 +1340,8 @@ static int update_peer_port_eid(struct ubagg_topo_info *new_topo_info, static int ubagg_update_topo_info(struct ubagg_topo_map *new_topo_map, struct ubagg_topo_map *old_topo_map) { - struct ubagg_topo_info *new_cur_node_info; - struct ubagg_topo_info *old_cur_node_info; + struct ubagg_topo_node *new_cur_node_info; + struct ubagg_topo_node *old_cur_node_info; uint32_t new_cur_node_index = 0; uint32_t old_cur_node_index = 0; @@ -1334,7 +1349,7 @@ static int ubagg_update_topo_info(struct ubagg_topo_map *new_topo_map, ubagg_log_err("Invalid topo map\n"); return -EINVAL; } - if (!is_bonding_and_primary_eid_valid(new_topo_map)) { + if (!is_aggr_and_primary_eid_valid(new_topo_map)) { ubagg_log_err("Invalid primary eid\n"); return -EINVAL; } @@ -1349,24 +1364,23 @@ static int ubagg_update_topo_info(struct ubagg_topo_map *new_topo_map, } old_cur_node_info = &(old_topo_map->topo_infos[old_cur_node_index]); - if (update_peer_port_eid(new_cur_node_info, old_cur_node_info) != 0) { - ubagg_log_err("update peer port eid failed\n"); - return -1; - } - return 0; + // 暂时先只更新拓扑连接关系，后续考虑更新eid + // TODO更新拓扑逻辑：只增不改 + update_dev_info(new_cur_node_info, old_cur_node_info); + return update_link_info(new_cur_node_info, old_cur_node_info); } -static bool has_add_dev_by_bonding_eid(const char *bonding_eid) +static bool has_add_dev_by_aggr_eid(const char *aggr_eid) { int i; - if (bonding_eid == NULL) { - ubagg_log_err("bonding_eid is NULL"); + if (aggr_eid == NULL) { + ubagg_log_err("aggr_eid is NULL"); return false; } mutex_lock(&g_name_eid_arr_lock); for (i = 0; i < UBAGG_MAX_BONDING_DEV_NUM; i++) { - if (compare_eids(bonding_eid, g_name_eid_arr[i].bonding_eid)) { + if (is_eid_match(aggr_eid, g_name_eid_arr[i].aggr_eid)) { mutex_unlock(&g_name_eid_arr_lock); return true; } @@ -1375,20 +1389,20 @@ static bool has_add_dev_by_bonding_eid(const char *bonding_eid) return false; } -static void fill_add_dev_cfg(struct ubagg_topo_info *topo_info, +static void fill_add_dev_cfg(struct ubagg_topo_aggr_dev *aggr_dev, struct ubagg_add_dev_by_uvs *arg) { int i, j, k; - (void)memcpy(&arg->bonding_eid, topo_info->bonding_eid, EID_LEN); + (void)memcpy(&arg->aggr_eid, aggr_dev->aggr_eid, EID_LEN); for (i = 0; i < IODIE_NUM; i++) (void)memcpy(&arg->slave_eid[i].primary_eid, - topo_info->io_die_info[i].primary_eid, EID_LEN); + aggr_dev->fe[i].primary_eid, EID_LEN); for (j = 0; j < IODIE_NUM; j++) { for (k = 0; k < MAX_PORT_NUM; k++) (void)memcpy(&arg->slave_eid[j].port_eid[k], - topo_info->io_die_info[j].port_eid[k], + aggr_dev->fe[j].port_eid[k], EID_LEN); } } @@ -1530,7 +1544,7 @@ static int init_ubagg_ubcore_dev(struct ubagg_device *ubagg_dev, ubagg_dev->ub_dev.eid_table.eid_entries[0].eid_index = 0; ubagg_dev->ub_dev.eid_table.eid_entries[0].net = &init_net; (void)memcpy(&ubagg_dev->ub_dev.eid_table.eid_entries[0].eid, - &arg->bonding_eid, UBAGG_EID_SIZE); + &arg->aggr_eid, UBAGG_EID_SIZE); ubagg_dev->ub_dev.eid_table.eid_entries[0].valid = true; return 0; @@ -1643,14 +1657,14 @@ static bool is_eid_empty(const char *eid) return true; } -static void find_add_master_dev(const char *bondingEid, const char *name) +static void find_add_master_dev(const char *aggr_eid, const char *name) { int i; int empty_index = -1; mutex_lock(&g_name_eid_arr_lock); for (i = 0; i < UBAGG_MAX_BONDING_DEV_NUM; i++) { - if (is_eid_empty(g_name_eid_arr[i].bonding_eid)) { + if (is_eid_empty(g_name_eid_arr[i].aggr_eid)) { empty_index = i; break; } @@ -1661,7 +1675,7 @@ static void find_add_master_dev(const char *bondingEid, const char *name) UBAGG_MAX_BONDING_DEV_NUM); return; } - (void)memcpy(g_name_eid_arr[empty_index].bonding_eid, bondingEid, + (void)memcpy(g_name_eid_arr[empty_index].aggr_eid, aggr_eid, EID_LEN); (void)snprintf(g_name_eid_arr[empty_index].master_dev_name, UBAGG_MAX_DEV_NAME_LEN, "%s", name); @@ -1670,10 +1684,12 @@ static void find_add_master_dev(const char *bondingEid, const char *name) static int ubagg_add_dev_by_uvs(struct ubagg_topo_map *topo_map) { - struct ubagg_topo_info *cur_node_info; + struct ubagg_topo_node *cur_node_info; struct ubagg_add_dev_by_uvs arg = { 0 }; char *master_dev_name = NULL; uint32_t cur_node_index = 0; + const char *aggr_eid = NULL; + int dev_id; if (find_cur_node_index(topo_map, &cur_node_index) != 0) { ubagg_log_err("find cur node index failed\n"); @@ -1681,75 +1697,103 @@ static int ubagg_add_dev_by_uvs(struct ubagg_topo_map *topo_map) } cur_node_info = &(topo_map->topo_infos[cur_node_index]); - if (has_add_dev_by_bonding_eid(cur_node_info->bonding_eid)) { - ubagg_log_info("has add dev by bonding eid: " EID_FMT "\n", - EID_RAW_ARGS(cur_node_info->bonding_eid)); - return 0; - } + // add DEV_NUM devs for every node + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (!is_aggr_dev_valid(&cur_node_info->devs[dev_id])) { + continue; + } + aggr_eid = cur_node_info->devs[dev_id].aggr_eid; + if (has_add_dev_by_aggr_eid(aggr_eid)) { + ubagg_log_info("has add dev by aggr eid: " EID_FMT "\n", + EID_RAW_ARGS(aggr_eid)); + continue; + } - master_dev_name = generate_master_dev_name(); - if (master_dev_name == NULL) { - ubagg_log_err("generate master dev name failed\n"); - return -1; - } + master_dev_name = generate_master_dev_name(); + if (master_dev_name == NULL) { + ubagg_log_err("generate master dev name failed\n"); + return -1; + } - (void)snprintf(arg.master_dev_name, UBAGG_MAX_DEV_NAME_LEN, "%s", - master_dev_name); - fill_add_dev_cfg(cur_node_info, &arg); + (void)snprintf(arg.master_dev_name, UBAGG_MAX_DEV_NAME_LEN, "%s", + master_dev_name); + fill_add_dev_cfg(&cur_node_info->devs[dev_id], &arg); - if (add_dev_by_uvs(&arg) != 0) { - release_bond_device_id_with_name(master_dev_name); + if (add_dev_by_uvs(&arg) != 0) { + // TODO：是否需要释放前面加入的设备？ + release_bond_device_id_with_name(master_dev_name); + kfree(master_dev_name); + ubagg_log_err("add ubagg dev by uvs failed\n"); + return -1; + } + find_add_master_dev(cur_node_info->devs[dev_id].aggr_eid, master_dev_name); kfree(master_dev_name); - ubagg_log_err("add ubagg dev by uvs failed\n"); - return -1; + master_dev_name = NULL; } - find_add_master_dev(cur_node_info->bonding_eid, master_dev_name); - kfree(master_dev_name); + return 0; } static void print_topo_map(struct ubagg_topo_map *topo_map) { - int i, j, k; - struct ubagg_topo_info *cur_node_info; - - ubagg_log_info( - "========================== topo map start =============================\n"); - for (i = 0; i < topo_map->node_num; i++) { - cur_node_info = topo_map->topo_infos + i; - if (is_eid_empty(cur_node_info->bonding_eid)) - continue; - - ubagg_log_info( - "===================== node %d start =======================\n", - i); - ubagg_log_info("bonding eid: " EID_FMT "\n", - EID_RAW_ARGS(cur_node_info->bonding_eid)); - for (j = 0; j < IODIE_NUM; j++) { - ubagg_log_info( - "\tprimary eid %d: " EID_FMT "\n", j, - EID_RAW_ARGS(cur_node_info->io_die_info[j] - .primary_eid)); - for (k = 0; k < MAX_PORT_NUM; k++) { - ubagg_log_info( - "\t\tport eid %d: " EID_FMT "\n", k, - EID_RAW_ARGS( - cur_node_info->io_die_info[j] - .port_eid[k])); - ubagg_log_info( - "\t\tpeer_port eid %d: " EID_FMT "\n", - k, - EID_RAW_ARGS( - cur_node_info->io_die_info[j] - .peer_port_eid[k])); + int node_idx, dev_idx, iodie_idx, port_idx; + struct ubagg_topo_node *node; + + if (!topo_map) { + ubagg_log_err("topo_map is NULL\n"); + return; + } + + ubagg_log_info( + "========================== topo map start =============================\n"); + for (node_idx = 0; node_idx < topo_map->node_num; node_idx++) { + node = &topo_map->topo_infos[node_idx]; + + ubagg_log_info( + "===================== node %d start =======================\n", + node_idx); + + /* print link table for this node */ + for (iodie_idx = 0; iodie_idx < IODIE_NUM; iodie_idx++) { + for (port_idx = 0; port_idx < MAX_PORT_NUM; port_idx++) { + ubagg_log_info("link[iodie_idx:%d][port_idx:%d] -> peer_node: %u, peer_iodie: %u, peer_port: %u\n", + iodie_idx, + port_idx, + node->links[iodie_idx][port_idx].peer_node, + node->links[iodie_idx][port_idx].peer_iodie, + node->links[iodie_idx][port_idx].peer_port); } } - ubagg_log_info( - "===================== node %d end =======================\n", - i); - } - ubagg_log_info( - "========================== topo map end =============================\n"); + + /* print device list (only devices with valid aggr_eid) */ + for (dev_idx = 0; dev_idx < DEV_NUM; dev_idx++) { + if (!is_eid_valid(node->devs[dev_idx].aggr_eid)) + continue; + + ubagg_log_info("---- dev %d aggr_eid: " EID_FMT "\n", dev_idx, + EID_RAW_ARGS(node->devs[dev_idx].aggr_eid)); + + for (iodie_idx = 0; iodie_idx < IODIE_NUM; iodie_idx++) { + ubagg_log_info("------ socket_id[%d]: %u\n", iodie_idx, + node->devs[dev_idx].fe[iodie_idx].socket_id); + + ubagg_log_info("------ primary_eid[%d]: " EID_FMT "\n", iodie_idx, + EID_RAW_ARGS(node->devs[dev_idx].fe[iodie_idx].primary_eid)); + + for (port_idx = 0; port_idx < MAX_PORT_NUM; port_idx++) { + ubagg_log_info("-------- port_eid[%d][%d]: " EID_FMT "\n", + iodie_idx, port_idx, + EID_RAW_ARGS(node->devs[dev_idx].fe[iodie_idx].port_eid[port_idx])); + } + } + } + + ubagg_log_info( + "===================== node %d end =======================\n", + node_idx); + } + ubagg_log_info( + "========================== topo map end =============================\n"); } static int ubagg_set_topo_info(struct ubagg_cmd_hdr *hdr) @@ -1785,7 +1829,7 @@ static int ubagg_set_topo_info(struct ubagg_cmd_hdr *hdr) ubagg_log_err("Failed to create topo map\n"); return -ENOMEM; } - if (!is_bonding_and_primary_eid_valid(topo_map)) { + if (!is_aggr_and_primary_eid_valid(topo_map)) { delete_global_ubagg_topo_map(); ubagg_log_err("Invalid primary eid\n"); return -EINVAL; diff --git a/drivers/ub/urma/ubagg/ubagg_ioctl.h b/drivers/ub/urma/ubagg/ubagg_ioctl.h index 5bf81e11c54a..3d1dbdc34d88 100644 --- a/drivers/ub/urma/ubagg/ubagg_ioctl.h +++ b/drivers/ub/urma/ubagg/ubagg_ioctl.h @@ -154,7 +154,7 @@ struct ubagg_slave_device { }; struct ubagg_topo_info_out { - struct ubagg_topo_info topo_info[MAX_NODE_NUM]; + struct ubagg_topo_node topo_info[MAX_NODE_NUM]; uint32_t node_num; }; @@ -167,7 +167,7 @@ struct ubagg_primary_port_eid { struct ubagg_add_dev_by_uvs { char master_dev_name[UBAGG_MAX_DEV_NAME_LEN]; - union ubcore_eid bonding_eid; + union ubcore_eid aggr_eid; struct ubagg_primary_port_eid slave_eid[IODIE_NUM]; }; diff --git a/drivers/ub/urma/ubagg/ubagg_topo_info.c b/drivers/ub/urma/ubagg/ubagg_topo_info.c index 8b8e60979a1a..7d823f9bc0b1 100644 --- a/drivers/ub/urma/ubagg/ubagg_topo_info.c +++ b/drivers/ub/urma/ubagg/ubagg_topo_info.c @@ -16,7 +16,7 @@ static struct ubagg_topo_map *g_topo_map; struct ubagg_topo_map * -create_global_ubagg_topo_map(struct ubagg_topo_info *topo_infos, +create_global_ubagg_topo_map(struct ubagg_topo_node *topo_infos, uint32_t node_num) { g_topo_map = create_ubagg_topo_map_from_user(topo_infos, node_num); @@ -37,7 +37,7 @@ struct ubagg_topo_map *get_global_ubagg_map(void) } struct ubagg_topo_map * -create_ubagg_topo_map_from_user(struct ubagg_topo_info *user_topo_infos, +create_ubagg_topo_map_from_user(struct ubagg_topo_node *user_topo_infos, uint32_t node_num) { struct ubagg_topo_map *topo_map = NULL; @@ -53,7 +53,7 @@ create_ubagg_topo_map_from_user(struct ubagg_topo_info *user_topo_infos, return NULL; ret = copy_from_user(topo_map->topo_infos, (void __user *)user_topo_infos, - sizeof(struct ubagg_topo_info) * node_num); + sizeof(struct ubagg_topo_node) * node_num); if (ret != 0) { ubagg_log_err("Failed to copy topo info\n"); kfree(topo_map); diff --git a/drivers/ub/urma/ubagg/ubagg_topo_info.h b/drivers/ub/urma/ubagg/ubagg_topo_info.h index d22b9b70e2b8..543bb32407cb 100644 --- a/drivers/ub/urma/ubagg/ubagg_topo_info.h +++ b/drivers/ub/urma/ubagg/ubagg_topo_info.h @@ -8,8 +8,8 @@ * Note: * History: 2025-06-07 Create file */ -#ifndef UBAGG_TOPO_INFO_H -#define UBAGG_TOPO_INFO_H +#ifndef ubagg_topo_info_H +#define ubagg_topo_info_H #include @@ -17,27 +17,39 @@ #define MAX_PORT_NUM (9) #define MAX_NODE_NUM (16) #define IODIE_NUM (2) +#define DEV_NUM (64) -struct ubagg_iodie_info { +struct ubagg_topo_fe { + uint32_t socket_id; char primary_eid[EID_LEN]; char port_eid[MAX_PORT_NUM][EID_LEN]; - char peer_port_eid[MAX_PORT_NUM][EID_LEN]; - int socket_id; }; -struct ubagg_topo_info { - char bonding_eid[EID_LEN]; - struct ubagg_iodie_info io_die_info[IODIE_NUM]; - bool is_cur_node; +struct ubagg_topo_aggr_dev { + char aggr_eid[EID_LEN]; + struct ubagg_topo_fe fe[IODIE_NUM]; +}; + +struct ubagg_topo_link { + uint32_t peer_node; // node id + uint32_t peer_iodie; // iodie idx + uint32_t peer_port; // port idx, UINT32_MAX indicates no connection +}; + +struct ubagg_topo_node { + uint32_t id; + uint32_t is_current; + struct ubagg_topo_link links[IODIE_NUM][MAX_PORT_NUM]; + struct ubagg_topo_aggr_dev devs[DEV_NUM]; }; struct ubagg_topo_map { - struct ubagg_topo_info topo_infos[MAX_NODE_NUM]; + struct ubagg_topo_node topo_infos[MAX_NODE_NUM]; uint32_t node_num; }; struct ubagg_topo_map * -create_global_ubagg_topo_map(struct ubagg_topo_info *topo_infos, +create_global_ubagg_topo_map(struct ubagg_topo_node *topo_infos, uint32_t node_num); void delete_global_ubagg_topo_map(void); @@ -45,8 +57,8 @@ void delete_global_ubagg_topo_map(void); struct ubagg_topo_map *get_global_ubagg_map(void); struct ubagg_topo_map * -create_ubagg_topo_map_from_user(struct ubagg_topo_info *topo_infos, +create_ubagg_topo_map_from_user(struct ubagg_topo_node *topo_infos, uint32_t node_num); void delete_ubagg_topo_map(struct ubagg_topo_map *topo_map); -#endif // UBAGG_TOPO_INFO_H +#endif // ubcore_topo_node_H diff --git a/drivers/ub/urma/ubcore/ubcore_cmd.h b/drivers/ub/urma/ubcore/ubcore_cmd.h index def379c2b9d5..4f75ba5a7a1e 100644 --- a/drivers/ub/urma/ubcore/ubcore_cmd.h +++ b/drivers/ub/urma/ubcore/ubcore_cmd.h @@ -127,7 +127,7 @@ struct ubcore_cmd_topo_info { } in; struct { uint32_t node_num; - struct ubcore_topo_info topo_info; + struct ubcore_topo_node topo_info; } out; }; diff --git a/drivers/ub/urma/ubcore/ubcore_connect_bonding.c b/drivers/ub/urma/ubcore/ubcore_connect_bonding.c index 56a0acac8d83..a15d15101e86 100644 --- a/drivers/ub/urma/ubcore/ubcore_connect_bonding.c +++ b/drivers/ub/urma/ubcore/ubcore_connect_bonding.c @@ -48,8 +48,9 @@ struct msg_jetty_info_resp { static struct ubcore_device *ubcore_find_physical_device(void) { struct ubcore_topo_map *topo_map; - struct ubcore_topo_info *topo_info; + struct ubcore_topo_node *topo_info; union ubcore_eid *primary_eid; + int dev_id; topo_map = ubcore_get_global_topo_map(); if (!topo_map) { @@ -63,15 +64,23 @@ static struct ubcore_device *ubcore_find_physical_device(void) return NULL; } - primary_eid = (union ubcore_eid *)topo_info->io_die_info[0].primary_eid; - return ubcore_find_device(primary_eid, UBCORE_TRANSPORT_UB); + // 首个非空设备的首个iodie的primary_eid + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (is_aggr_dev_valid(&topo_info->devs[dev_id])) { + primary_eid = (union ubcore_eid *)topo_info->devs[dev_id].fe[0].primary_eid; + return ubcore_find_device(primary_eid, UBCORE_TRANSPORT_UB); + } + } + + return NULL; } static struct ubcore_device *ubcore_find_bonding_device(void) { struct ubcore_topo_map *topo_map; - struct ubcore_topo_info *topo_info; - union ubcore_eid *bonding_eid; + struct ubcore_topo_node *topo_info; + union ubcore_eid *aggr_eid; + int dev_id; topo_map = ubcore_get_global_topo_map(); if (!topo_map) { @@ -85,8 +94,13 @@ static struct ubcore_device *ubcore_find_bonding_device(void) return NULL; } - bonding_eid = (union ubcore_eid *)topo_info->bonding_eid; - return ubcore_find_device(bonding_eid, UBCORE_TRANSPORT_UB); + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (is_aggr_dev_valid(&topo_info->devs[dev_id])) { + aggr_eid = (union ubcore_eid *)topo_info->devs[dev_id].aggr_eid; + return ubcore_find_device(aggr_eid, UBCORE_TRANSPORT_UB); + } + } + return NULL; } static struct ubcore_session * @@ -128,7 +142,7 @@ static int send_seg_info_req(struct ubcore_device *dev, uint32_t session_id, msg.session_id = session_id; msg.data = req; - ret = ubcore_get_primary_eid_by_bonding_eid(&req->ubva.eid, &dest_eid); + ret = ubcore_get_primary_eid_by_aggr_eid(&req->ubva.eid, &dest_eid); if (ret != 0) return ret; @@ -174,7 +188,7 @@ static int send_jetty_info_req(struct ubcore_device *dev, uint32_t session_id, msg.session_id = session_id; msg.data = req; - ret = ubcore_get_primary_eid_by_bonding_eid(&req->jetty_id.eid, + ret = ubcore_get_primary_eid_by_aggr_eid(&req->jetty_id.eid, &dest_eid); if (ret != 0) return ret; diff --git a/drivers/ub/urma/ubcore/ubcore_genl_admin.c b/drivers/ub/urma/ubcore/ubcore_genl_admin.c index b04e81f94e54..62efd103f9c1 100644 --- a/drivers/ub/urma/ubcore/ubcore_genl_admin.c +++ b/drivers/ub/urma/ubcore/ubcore_genl_admin.c @@ -307,7 +307,7 @@ int ubcore_get_topo_info(struct sk_buff *skb, struct genl_info *info) arg.out.node_num = topo_map->node_num; (void)memcpy(&arg.out.topo_info, &topo_map->topo_infos[arg.in.node_idx], - sizeof(struct ubcore_topo_info)); + sizeof(struct ubcore_topo_node)); return ubcore_copy_to_user((void __user *)(uintptr_t)args_addr, &arg, sizeof(struct ubcore_cmd_topo_info)); } diff --git a/drivers/ub/urma/ubcore/ubcore_topo_info.c b/drivers/ub/urma/ubcore/ubcore_topo_info.c index e9f56ff09500..eed0acae8b32 100644 --- a/drivers/ub/urma/ubcore/ubcore_topo_info.c +++ b/drivers/ub/urma/ubcore/ubcore_topo_info.c @@ -19,7 +19,7 @@ static struct ubcore_topo_map *g_ubcore_topo_map; struct ubcore_topo_map * -ubcore_create_global_topo_map(struct ubcore_topo_info *topo_infos, +ubcore_create_global_topo_map(struct ubcore_topo_node *topo_infos, uint32_t node_num) { g_ubcore_topo_map = @@ -41,7 +41,7 @@ struct ubcore_topo_map *ubcore_get_global_topo_map(void) } struct ubcore_topo_map * -ubcore_create_topo_map_from_user(struct ubcore_topo_info *user_topo_infos, +ubcore_create_topo_map_from_user(struct ubcore_topo_node *user_topo_infos, uint32_t node_num) { struct ubcore_topo_map *topo_map = NULL; @@ -57,7 +57,7 @@ ubcore_create_topo_map_from_user(struct ubcore_topo_info *user_topo_infos, return NULL; ret = copy_from_user(topo_map->topo_infos, (void __user *)user_topo_infos, - sizeof(struct ubcore_topo_info) * node_num); + sizeof(struct ubcore_topo_node) * node_num); if (ret != 0) { ubcore_log_err("Failed to copy topo infos\n"); kfree(topo_map); @@ -74,6 +74,13 @@ void ubcore_delete_topo_map(struct ubcore_topo_map *topo_map) kfree(topo_map); } +bool is_aggr_dev_valid(struct ubcore_topo_aggr_dev *aggr_dev) +{ + struct ubcore_topo_aggr_dev empty_dev = {0}; + + return (memcmp(aggr_dev, &empty_dev, sizeof(struct ubcore_topo_aggr_dev)) == 0) ? false : true; +} + bool is_eid_valid(const char *eid) { int i; @@ -85,23 +92,23 @@ bool is_eid_valid(const char *eid) return false; } -bool is_bonding_and_primary_eid_valid(struct ubcore_topo_map *topo_map) +bool is_aggr_and_primary_eid_valid(struct ubcore_topo_map *topo_map) { - int i, j; + int i, j, k; bool has_primary_eid = false; for (i = 0; i < topo_map->node_num; i++) { - if (!is_eid_valid(topo_map->topo_infos[i].bonding_eid)) - return false; - has_primary_eid = false; - for (j = 0; j < IODIE_NUM; j++) { - if (is_eid_valid(topo_map->topo_infos[i] - .io_die_info[j] - .primary_eid)) - has_primary_eid = true; + for (j = 0; j < DEV_NUM; j++) { + if (!is_eid_valid(topo_map->topo_infos[i].devs[j].aggr_eid)) + return false; + has_primary_eid = false; + for (k = 0; k < IODIE_NUM; k++) { + if (is_eid_valid(topo_map->topo_infos[i].devs[j].fe[k].primary_eid)) + has_primary_eid = true; + } + if (!has_primary_eid) + return false; } - if (!has_primary_eid) - return false; } return true; } @@ -112,7 +119,7 @@ static int find_cur_node_index(struct ubcore_topo_map *topo_map, int i; for (i = 0; i < topo_map->node_num; i++) { - if (topo_map->topo_infos[i].is_cur_node) { + if (topo_map->topo_infos[i].is_current) { *node_index = i; break; } @@ -124,49 +131,12 @@ static int find_cur_node_index(struct ubcore_topo_map *topo_map, return 0; } -static bool compare_eids(const char *eid1, const char *eid2) +static bool is_eid_match(const char *eid1, const char *eid2) { return memcmp(eid1, eid2, EID_LEN) == 0; } -static int update_peer_port_eid(struct ubcore_topo_info *new_topo_info, - struct ubcore_topo_info *old_topo_info) -{ - int i, j; - char *new_peer_port_eid; - char *old_peer_port_eid; - - for (i = 0; i < IODIE_NUM; i++) { - for (j = 0; j < MAX_PORT_NUM; j++) { - if (!is_eid_valid( - new_topo_info->io_die_info[i].port_eid[j])) - continue; - - new_peer_port_eid = - new_topo_info->io_die_info[i].peer_port_eid[j]; - old_peer_port_eid = - old_topo_info->io_die_info[i].peer_port_eid[j]; - - if (!is_eid_valid(new_peer_port_eid)) - continue; - if (is_eid_valid(old_peer_port_eid) && - !compare_eids(new_peer_port_eid, - old_peer_port_eid)) { - ubcore_log_err( - "peer port eid is not same, new: " EID_FMT - ", old: " EID_FMT "\n", - EID_RAW_ARGS(new_peer_port_eid), - EID_RAW_ARGS(old_peer_port_eid)); - return -EINVAL; - } - (void)memcpy(old_peer_port_eid, new_peer_port_eid, - EID_LEN); - } - } - return 0; -} - -struct ubcore_topo_info * +struct ubcore_topo_node * ubcore_get_cur_topo_info(struct ubcore_topo_map *topo_map) { uint32_t cur_node_index = 0; @@ -178,11 +148,57 @@ ubcore_get_cur_topo_info(struct ubcore_topo_map *topo_map) return &(topo_map->topo_infos[cur_node_index]); } +static void update_dev_info(struct ubcore_topo_node *new_topo_info, + struct ubcore_topo_node *old_topo_info) +{ + int dev_id; + + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (is_aggr_dev_valid(&old_topo_info->devs[dev_id])) { + (void)memcpy(&old_topo_info->devs[dev_id], &new_topo_info->devs[dev_id], + sizeof(struct ubcore_topo_aggr_dev)); + } + // TODO: update eid + } +} + +static int update_link_info(struct ubcore_topo_node *new_topo_info, + struct ubcore_topo_node *old_topo_info) +{ + int iodie_id, port_id, old_remote_port_id; + + for (iodie_id = 0; iodie_id < IODIE_NUM; iodie_id++) { + for (port_id = 0; port_id < PORT_NUM; ++port_id) { + // add new connection if no connection exists + old_remote_port_id = old_topo_info->links[iodie_id][port_id].peer_port; + if (old_remote_port_id == UINT32_MAX) { + (void)memcpy(&old_topo_info->links[iodie_id][port_id], + &new_topo_info->links[iodie_id][port_id], sizeof(struct ubcore_topo_link)); + } else { + if (memcmp(&old_topo_info->links[iodie_id][port_id], &new_topo_info->links[iodie_id][port_id], + sizeof(struct ubcore_topo_link)) != 0) { + ubcore_log_err("link is not the same, new: peer_node[%u]/peer_iodie[%u]/peer_port[%u],\ + old: peer_node[%u]/peer_iodie[%u]/peer_port[%u]\n", + new_topo_info->links[iodie_id][port_id].peer_node, + new_topo_info->links[iodie_id][port_id].peer_iodie, + new_topo_info->links[iodie_id][port_id].peer_port, + old_topo_info->links[iodie_id][port_id].peer_node, + old_topo_info->links[iodie_id][port_id].peer_iodie, + old_topo_info->links[iodie_id][port_id].peer_port); + return -EINVAL; + } + } + } + } + return 0; +} + +// only update current node int ubcore_update_topo_map(struct ubcore_topo_map *new_topo_map, struct ubcore_topo_map *old_topo_map) { - struct ubcore_topo_info *new_cur_node_info; - struct ubcore_topo_info *old_cur_node_info; + struct ubcore_topo_node *new_cur_node_info; + struct ubcore_topo_node *old_cur_node_info; uint32_t new_cur_node_index = 0; uint32_t old_cur_node_index = 0; @@ -190,7 +206,7 @@ int ubcore_update_topo_map(struct ubcore_topo_map *new_topo_map, ubcore_log_err("Invalid topo map\n"); return -EINVAL; } - if (!is_bonding_and_primary_eid_valid(new_topo_map)) { + if (!is_aggr_and_primary_eid_valid(new_topo_map)) { ubcore_log_err("Invalid primary eid\n"); return -EINVAL; } @@ -205,61 +221,78 @@ int ubcore_update_topo_map(struct ubcore_topo_map *new_topo_map, } old_cur_node_info = &(old_topo_map->topo_infos[old_cur_node_index]); - if (update_peer_port_eid(new_cur_node_info, old_cur_node_info) != 0) { - ubcore_log_err("update peer port eid failed\n"); - return -EINVAL; - } - return 0; + // 暂时先只更新拓扑连接关系，后续考虑更新eid + // TODO更新拓扑逻辑：只增不改 + update_dev_info(new_cur_node_info, old_cur_node_info); + return update_link_info(new_cur_node_info, old_cur_node_info); } void ubcore_show_topo_map(struct ubcore_topo_map *topo_map) { - int i, j, k; - struct ubcore_topo_info *cur_node_info; - - ubcore_log_info( - "========================== topo map start =============================\n"); - for (i = 0; i < topo_map->node_num; i++) { - cur_node_info = topo_map->topo_infos + i; - if (!is_eid_valid(cur_node_info->bonding_eid)) - continue; - - ubcore_log_info( - "===================== node %d start =======================\n", - i); - ubcore_log_info("bonding eid: " EID_FMT "\n", - EID_RAW_ARGS(cur_node_info->bonding_eid)); - for (j = 0; j < IODIE_NUM; j++) { - ubcore_log_info( - "**primary eid %d: " EID_FMT "\n", j, - EID_RAW_ARGS(cur_node_info->io_die_info[j] - .primary_eid)); - for (k = 0; k < MAX_PORT_NUM; k++) { - ubcore_log_info( - "****port eid %d: " EID_FMT "\n", k, - EID_RAW_ARGS( - cur_node_info->io_die_info[j] - .port_eid[k])); - ubcore_log_info( - "****peer_port eid %d: " EID_FMT "\n", - k, - EID_RAW_ARGS( - cur_node_info->io_die_info[j] - .peer_port_eid[k])); + int node_idx, dev_idx, die_idx, port_idx; + struct ubcore_topo_node *node; + + if (!topo_map) { + ubcore_log_err("topo_map is NULL\n"); + return; + } + + ubcore_log_info( + "========================== topo map start =============================\n"); + for (node_idx = 0; node_idx < topo_map->node_num; node_idx++) { + node = &topo_map->topo_infos[node_idx]; + + ubcore_log_info( + "===================== node %d start =======================\n", + node_idx); + + /* print link table for this node */ + for (die_idx = 0; die_idx < IODIE_NUM; die_idx++) { + for (port_idx = 0; port_idx < PORT_NUM; port_idx++) { + ubcore_log_info("link[iodie_idx:%d][port_idx:%d] -> peer_node: %u, peer_iodie: %u, peer_port: %u\n", + die_idx, + port_idx, + node->links[die_idx][port_idx].peer_node, + node->links[die_idx][port_idx].peer_iodie, + node->links[die_idx][port_idx].peer_port); } - } - ubcore_log_info( - "===================== node %d end =======================\n", - i); - } - ubcore_log_info( - "========================== topo map end =============================\n"); + } + + /* print device list (only devices with valid aggr_eid) */ + for (dev_idx = 0; dev_idx < DEV_NUM; dev_idx++) { + if (!is_eid_valid(node->devs[dev_idx].aggr_eid)) + continue; + + ubcore_log_info("---- dev %d aggr_eid: " EID_FMT "\n", dev_idx, + EID_RAW_ARGS(node->devs[dev_idx].aggr_eid)); + + for (die_idx = 0; die_idx < IODIE_NUM; die_idx++) { + ubcore_log_info("------ socket_id[%d]: %u\n", die_idx, + node->devs[dev_idx].fe[die_idx].socket_id); + + ubcore_log_info("------ primary_eid[%d]: " EID_FMT "\n", die_idx, + EID_RAW_ARGS(node->devs[dev_idx].fe[die_idx].primary_eid)); + + for (port_idx = 0; port_idx < PORT_NUM; port_idx++) { + ubcore_log_info("-------- port_eid[%d][%d]: " EID_FMT "\n", + die_idx, port_idx, + EID_RAW_ARGS(node->devs[dev_idx].fe[die_idx].port_eid[port_idx])); + } + } + } + + ubcore_log_info( + "===================== node %d end =======================\n", + node_idx); + } + ubcore_log_info( + "========================== topo map end =============================\n"); } int ubcore_get_primary_eid(union ubcore_eid *eid, union ubcore_eid *primary_eid) { - int i, j, k; - struct ubcore_topo_info *cur_node_info; + int node_id, dev_id, iodie_id, port_id; + struct ubcore_topo_node *cur_node_info; if (!g_ubcore_topo_map) { ubcore_log_info( @@ -268,37 +301,36 @@ int ubcore_get_primary_eid(union ubcore_eid *eid, union ubcore_eid *primary_eid) return 0; } - for (i = 0; i < g_ubcore_topo_map->node_num; i++) { - cur_node_info = g_ubcore_topo_map->topo_infos + i; - if (compare_eids(cur_node_info->bonding_eid, + for (node_id = 0; node_id < g_ubcore_topo_map->node_num; node_id++) { + cur_node_info = g_ubcore_topo_map->topo_infos + node_id; + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (!is_eid_match(cur_node_info->devs[dev_id].aggr_eid, (char *)eid->raw)) { - ubcore_log_err("input eid is bonding eid\n"); - return -EINVAL; - } - for (j = 0; j < IODIE_NUM; j++) { - if (compare_eids( - cur_node_info->io_die_info[j].primary_eid, - (char *)eid->raw)) { - (void)memcpy(primary_eid, - cur_node_info->io_die_info[j] - .primary_eid, - EID_LEN); - ubcore_log_info("input eid is primary eid\n"); - return 0; + ubcore_log_err("failed to get primary eid using aggr eid\n"); + return -EINVAL; } - for (k = 0; k < MAX_PORT_NUM; k++) { - if (compare_eids(cur_node_info->io_die_info[j] - .port_eid[k], - (char *)eid->raw)) { + + for (iodie_id = 0; iodie_id < IODIE_NUM; iodie_id++) { + if (is_eid_match( + cur_node_info->devs[dev_id].fe[iodie_id].primary_eid, + (char *)eid->raw)) { (void)memcpy(primary_eid, - cur_node_info - ->io_die_info[j] - .primary_eid, - EID_LEN); - ubcore_log_info( - "find primary eid by port eid\n"); + cur_node_info->devs[dev_id].fe[iodie_id].primary_eid, + EID_LEN); + ubcore_log_info("find primary eid\n"); return 0; } + for (port_id = 0; port_id < MAX_PORT_NUM; port_id++) { + if (is_eid_match(cur_node_info->devs[dev_id].fe[iodie_id].port_eid[port_id], + (char *)eid->raw)) { + (void)memcpy(primary_eid, + cur_node_info->devs[dev_id].fe[iodie_id].primary_eid, + EID_LEN); + ubcore_log_info( + "find primary eid by port eid\n"); + return 0; + } + } } } } @@ -306,73 +338,75 @@ int ubcore_get_primary_eid(union ubcore_eid *eid, union ubcore_eid *primary_eid) return -EINVAL; } -static struct ubcore_topo_info * - ubcore_get_topo_info_by_bonding_eid(union ubcore_eid *bonding_eid) +static struct ubcore_topo_node * + ubcore_get_topo_info_by_aggr_eid(union ubcore_eid *aggr_eid, int *device_id) { struct ubcore_topo_map *topo_map; - int i; + int node_id, dev_id; topo_map = g_ubcore_topo_map; - for (i = 0; i < topo_map->node_num; i++) { - if (!memcmp(bonding_eid, topo_map->topo_infos[i].bonding_eid, - sizeof(*bonding_eid))) - return &topo_map->topo_infos[i]; + for (node_id = 0; node_id < topo_map->node_num; node_id++) { + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (memcmp(aggr_eid, topo_map->topo_infos[node_id].devs[dev_id].aggr_eid, + sizeof(*aggr_eid)) == 0) { + *device_id = dev_id; + return &topo_map->topo_infos[node_id]; + } + } } ubcore_log_err( - "Failed to get topo info, bonding_eid: "EID_FMT".\n", - EID_ARGS(*bonding_eid)); + "Failed to get topo info, aggr_eid: "EID_FMT".\n", + EID_ARGS(*aggr_eid)); return NULL; } static int ubcore_get_route_port_eid(union ubcore_eid *src_v_eid, union ubcore_eid *dst_v_eid, struct ubcore_route_list *route_list) { - struct ubcore_topo_info *src_topo_info = NULL; - struct ubcore_topo_info *dst_topo_info = NULL; + int src_dev_id, dst_dev_id, iodie_id, port_id, remote_port_id; + struct ubcore_topo_aggr_dev *src_aggr_dev = NULL; + struct ubcore_topo_aggr_dev *dst_aggr_dev = NULL; + struct ubcore_topo_node *src_topo_info = NULL; + struct ubcore_topo_node *dst_topo_info = NULL; uint32_t num = route_list->route_num; - int i, j; - src_topo_info = - ubcore_get_topo_info_by_bonding_eid(src_v_eid); + src_topo_info = ubcore_get_topo_info_by_aggr_eid(src_v_eid, &src_dev_id); if (IS_ERR_OR_NULL(src_topo_info)) { ubcore_log_err("Failed to get src_topo_info.\n"); return -EINVAL; } + src_aggr_dev = &src_topo_info->devs[src_dev_id]; - dst_topo_info = - ubcore_get_topo_info_by_bonding_eid(dst_v_eid); + dst_topo_info = ubcore_get_topo_info_by_aggr_eid(dst_v_eid, &dst_dev_id); if (IS_ERR_OR_NULL(dst_topo_info)) { ubcore_log_err("Failed to get dst_topo_info.\n"); return -EINVAL; } + dst_aggr_dev = &dst_topo_info->devs[dst_dev_id]; - for (int k = 0; k < IODIE_NUM; k++) { - for (i = 0; i < MAX_PORT_NUM; i++) { - if (!is_eid_valid(src_topo_info->io_die_info[k].port_eid[i]) || - !is_eid_valid(src_topo_info->io_die_info[k].peer_port_eid[i])) { - continue; + for (iodie_id = 0; iodie_id < IODIE_NUM; iodie_id++) { + for (port_id = 0; port_id < MAX_PORT_NUM; port_id++) { + if (num >= UBCORE_MAX_ROUTE_NUM) { + ubcore_log_warn("Invalid route num.\n"); + return -EINVAL; } - for (j = 0; j < MAX_PORT_NUM; j++) { - if (num >= UBCORE_MAX_ROUTE_NUM) { - ubcore_log_warn("Invalid route num.\n"); - return -EINVAL; - } - if (compare_eids( - src_topo_info->io_die_info[k].peer_port_eid[i], - dst_topo_info->io_die_info[k].port_eid[j])) { - (void)memcpy(&route_list->buf[num].src, - src_topo_info->io_die_info[k].port_eid[i], - sizeof(union ubcore_eid)); - (void)memcpy(&route_list->buf[num].dst, - src_topo_info->io_die_info[k].peer_port_eid[i], - sizeof(union ubcore_eid)); - route_list->buf[num].flag.bs.rtp = 1; - route_list->buf[num].flag.bs.ctp = 1; - route_list->buf[num].flag.bs.utp = 1; - num++; - } + if (!is_eid_valid(src_aggr_dev->fe[iodie_id].port_eid[port_id]) || + src_topo_info->links[iodie_id][port_id].peer_port == UINT32_MAX) { + continue; } + // use link to get peer info + remote_port_id = src_topo_info->links[iodie_id][port_id].peer_port; + (void)memcpy(&route_list->buf[num].src, + src_aggr_dev->fe[iodie_id].port_eid[port_id], + sizeof(union ubcore_eid)); + (void)memcpy(&route_list->buf[num].dst, + dst_aggr_dev->fe[iodie_id].port_eid[remote_port_id], + sizeof(union ubcore_eid)); + route_list->buf[num].flag.bs.rtp = 1; + route_list->buf[num].flag.bs.ctp = 1; + route_list->buf[num].flag.bs.utp = 1; + num++; } } @@ -386,11 +420,12 @@ static int ubcore_get_route_port_eid(union ubcore_eid *src_v_eid, return 0; } -int ubcore_get_primary_eid_by_bonding_eid(union ubcore_eid *bonding_eid, +// 默认选择0号iodie +int ubcore_get_primary_eid_by_aggr_eid(union ubcore_eid *aggr_eid, union ubcore_eid *primary_eid) { struct ubcore_topo_map *topo_map; - int i; + int node_id, dev_id; topo_map = ubcore_get_global_topo_map(); if (!topo_map) { @@ -398,66 +433,74 @@ int ubcore_get_primary_eid_by_bonding_eid(union ubcore_eid *bonding_eid, return -EINVAL; } - for (i = 0; i < topo_map->node_num; i++) { - if (!memcmp(bonding_eid, topo_map->topo_infos[i].bonding_eid, - sizeof(*bonding_eid))) { - *primary_eid = *((union ubcore_eid *) - topo_map->topo_infos[i].io_die_info[0].primary_eid); - return 0; + for (node_id = 0; node_id < topo_map->node_num; node_id++) { + for (dev_id = 0; dev_id < DEV_NUM; dev_id++) { + if (memcmp(aggr_eid, topo_map->topo_infos[node_id].devs[dev_id].aggr_eid, + sizeof(*aggr_eid)) == 0) { + *primary_eid = *((union ubcore_eid *) + topo_map->topo_infos[node_id].devs[dev_id].fe[0].primary_eid); + return 0; + } } } return -EINVAL; } +// 路由是什么粒度？---通过aggr_eid定位dev，以primary eid对为单位进行路由 static int ubcore_get_route_primary_eid(union ubcore_eid *src_v_eid, union ubcore_eid *dst_v_eid, struct ubcore_route_list *route_list) { + int src_dev_id, dst_dev_id, iodie_id, port_id, remote_port_id; + struct ubcore_topo_aggr_dev *src_aggr_dev = NULL; + struct ubcore_topo_aggr_dev *dst_aggr_dev = NULL; + struct ubcore_topo_node *src_topo_info = NULL; + struct ubcore_topo_node *dst_topo_info = NULL; uint32_t num = route_list->route_num; - struct ubcore_topo_info *topo_info = g_ubcore_topo_map->topo_infos; - bool src_match = false; - bool dst_match = false; - - for (int i = 0; i < g_ubcore_topo_map->node_num; i++) { - if (num >= UBCORE_MAX_ROUTE_NUM - 1) { - ubcore_log_warn("Finish to query topo map.\n"); - return -EINVAL; - } - if (!memcmp(src_v_eid, topo_info[i].bonding_eid, - sizeof(*src_v_eid))) { - route_list->buf[num].flag.bs.ctp = 1; - route_list->buf[num + 1].flag.bs.ctp = 1; - route_list->buf[num].hops = 0; - route_list->buf[num + 1].hops = 0; - (void)memcpy(&route_list->buf[num].src, - topo_info[i].io_die_info[0].primary_eid, - sizeof(union ubcore_eid)); - (void)memcpy(&route_list->buf[num + 1].src, - topo_info[i].io_die_info[1].primary_eid, - sizeof(union ubcore_eid)); - src_match = true; - } - if (!memcmp(dst_v_eid, topo_info[i].bonding_eid, - sizeof(*dst_v_eid))) { - route_list->buf[num].flag.bs.ctp = 1; - route_list->buf[num + 1].flag.bs.ctp = 1; - route_list->buf[num].hops = 0; - route_list->buf[num + 1].hops = 0; - (void)memcpy(&route_list->buf[num].dst, - topo_info[i].io_die_info[0].primary_eid, - sizeof(union ubcore_eid)); - (void)memcpy(&route_list->buf[num + 1].dst, - topo_info[i].io_die_info[1].primary_eid, - sizeof(union ubcore_eid)); - dst_match = true; + bool has_link = false; + + src_topo_info = ubcore_get_topo_info_by_aggr_eid(src_v_eid, &src_dev_id); + if (IS_ERR_OR_NULL(src_topo_info)) { + ubcore_log_err("Failed to get src_topo_info.\n"); + return -EINVAL; + } + src_aggr_dev = &src_topo_info->devs[src_dev_id]; + + dst_topo_info = ubcore_get_topo_info_by_aggr_eid(dst_v_eid, &dst_dev_id); + if (IS_ERR_OR_NULL(dst_topo_info)) { + ubcore_log_err("Failed to get dst_topo_info.\n"); + return -EINVAL; + } + dst_aggr_dev = &dst_topo_info->devs[dst_dev_id]; + + // update route---check if physical link exists? + for (iodie_id = 0; iodie_id < IODIE_NUM; iodie_id++) { + for (port_id = 0; port_id < PORT_NUM; port_id++) { + remote_port_id = src_topo_info->links[iodie_id][port_id].peer_port; + if (remote_port_id == UINT32_MAX) { + continue; + } + if (src_topo_info->links[iodie_id][port_id].peer_node == dst_topo_info->id) { + has_link = true; + break; + } } - if (src_match && dst_match) { - num += 2; + if (has_link == true) { break; } } + if (has_link == false) { + ubcore_log_err("No links to dst eid.\n"); + return -EINVAL; + } + for (iodie_id = 0; iodie_id < IODIE_NUM; iodie_id++) { + route_list->buf[num + iodie_id].flag.bs.ctp = 1; + route_list->buf[num + iodie_id].hops = 0; + (void)memcpy(&route_list->buf[num + iodie_id].src, src_aggr_dev->fe[iodie_id].primary_eid, sizeof(union ubcore_eid)); + (void)memcpy(&route_list->buf[num + iodie_id].dst, dst_aggr_dev->fe[iodie_id].primary_eid, sizeof(union ubcore_eid)); + } - route_list->route_num = num; - return (num == 0) ? -EINVAL : 0; + route_list->route_num += IODIE_NUM; + return 0; } int ubcore_get_route_list(struct ubcore_route *route, diff --git a/drivers/ub/urma/ubcore/ubcore_topo_info.h b/drivers/ub/urma/ubcore/ubcore_topo_info.h index 26d573b25609..5401bfc50c01 100644 --- a/drivers/ub/urma/ubcore/ubcore_topo_info.h +++ b/drivers/ub/urma/ubcore/ubcore_topo_info.h @@ -18,37 +18,51 @@ #define MAX_PORT_NUM (9) #define MAX_NODE_NUM (16) #define IODIE_NUM (2) +#define PORT_NUM (9) +#define DEV_NUM (64) -struct ubcore_iodie_info { +struct ubcore_topo_fe { + uint32_t socket_id; char primary_eid[EID_LEN]; - char port_eid[MAX_PORT_NUM][EID_LEN]; - char peer_port_eid[MAX_PORT_NUM][EID_LEN]; - int socket_id; + char port_eid[PORT_NUM][EID_LEN]; }; -struct ubcore_topo_info { - char bonding_eid[EID_LEN]; - struct ubcore_iodie_info io_die_info[IODIE_NUM]; - bool is_cur_node; +struct ubcore_topo_aggr_dev { + char aggr_eid[EID_LEN]; + struct ubcore_topo_fe fe[IODIE_NUM]; +}; + +struct ubcore_topo_link { + uint32_t peer_node; // node id + uint32_t peer_iodie; // iodie idx + uint32_t peer_port; // port idx, UINT32_MAX indicates no connection +}; + +struct ubcore_topo_node { + uint32_t id; + uint32_t is_current; + struct ubcore_topo_link links[IODIE_NUM][PORT_NUM]; + struct ubcore_topo_aggr_dev devs[DEV_NUM]; }; struct ubcore_topo_map { - struct ubcore_topo_info topo_infos[MAX_NODE_NUM]; + struct ubcore_topo_node topo_infos[MAX_NODE_NUM]; uint32_t node_num; }; struct ubcore_topo_map * -ubcore_create_global_topo_map(struct ubcore_topo_info *topo_infos, +ubcore_create_global_topo_map(struct ubcore_topo_node *topo_infos, uint32_t node_num); void ubcore_delete_global_topo_map(void); struct ubcore_topo_map *ubcore_get_global_topo_map(void); struct ubcore_topo_map * -ubcore_create_topo_map_from_user(struct ubcore_topo_info *user_topo_infos, +ubcore_create_topo_map_from_user(struct ubcore_topo_node *user_topo_infos, uint32_t node_num); void ubcore_delete_topo_map(struct ubcore_topo_map *topo_map); +bool is_aggr_dev_valid(struct ubcore_topo_aggr_dev *aggr_dev); bool is_eid_valid(const char *eid); -bool is_bonding_and_primary_eid_valid(struct ubcore_topo_map *topo_map); -struct ubcore_topo_info * +bool is_aggr_and_primary_eid_valid(struct ubcore_topo_map *topo_map); +struct ubcore_topo_node * ubcore_get_cur_topo_info(struct ubcore_topo_map *topo_map); int ubcore_update_topo_map(struct ubcore_topo_map *new_topo_map, struct ubcore_topo_map *old_topo_map); @@ -56,7 +70,7 @@ void ubcore_show_topo_map(struct ubcore_topo_map *topo_map); int ubcore_get_primary_eid(union ubcore_eid *eid, union ubcore_eid *primary_eid); -int ubcore_get_primary_eid_by_bonding_eid(union ubcore_eid *bonding_eid, +int ubcore_get_primary_eid_by_aggr_eid(union ubcore_eid *aggr_eid, union ubcore_eid *primary_eid); #endif // UBCORE_TOPO_INFO_H diff --git a/drivers/ub/urma/ubcore/ubcore_uvs_cmd.c b/drivers/ub/urma/ubcore/ubcore_uvs_cmd.c index e33be9653314..e043be12eca6 100644 --- a/drivers/ub/urma/ubcore/ubcore_uvs_cmd.c +++ b/drivers/ub/urma/ubcore/ubcore_uvs_cmd.c @@ -172,57 +172,63 @@ static int ubcore_get_eid_index(struct ubcore_device *dev, static int ubcore_create_jetty_rsrc(struct ubcore_topo_map *topo_map) { - struct ubcore_device *dev; - struct ubcore_eid_info eid_info = { 0 }; - struct ubcore_topo_info *cur_node_info; - int i, ret; + struct ubcore_device *dev; + struct ubcore_eid_info eid_info = { 0 }; + struct ubcore_topo_node *node; bool has_any_primary_eid = false; - - cur_node_info = ubcore_get_cur_topo_info(topo_map); - if (cur_node_info == NULL) { - ubcore_log_err("Failed to get cur node info\n"); - return -EINVAL; - } - - for (i = 0; i < IODIE_NUM; i++) { - if (!is_eid_valid(cur_node_info->io_die_info[i].primary_eid)) - continue; - has_any_primary_eid = true; - (void)memcpy(&eid_info.eid, - cur_node_info->io_die_info[i].primary_eid, - sizeof(union ubcore_eid)); - dev = ubcore_get_device_by_eid(&eid_info.eid, - UBCORE_TRANSPORT_UB); - if (dev == NULL) { - ubcore_log_err( - "primary %d dev not exist, eid: " EID_FMT "\n", - i, - EID_RAW_ARGS(cur_node_info->io_die_info[i] - .primary_eid)); - return -1; - } - - ret = ubcore_get_eid_index(dev, &eid_info.eid, - &eid_info.eid_index); - if (ret != 0) { - ubcore_log_err("Failed to get eid index\n"); - return ret; - } - - ret = ubcore_call_cm_eid_ops(dev, &eid_info, - UBCORE_MGMT_EVENT_EID_ADD); - if (ret != 0) { - ubcore_log_err("Failed to call cm eid ops\n"); - return ret; - } - ubcore_log_info( - "Success to create jetty rsrc: primary %d dev %s, eid: " EID_FMT - ", idx: %d\n", - i, dev->dev_name, - EID_RAW_ARGS(cur_node_info->io_die_info[i].primary_eid), - eid_info.eid_index); - } - return has_any_primary_eid ? 0 : -1; + int node_idx, dev_idx, die_idx; + int ret; + + if (!topo_map) { + ubcore_log_err("topo_map is NULL\n"); + return -EINVAL; + } + + for (node_idx = 0; node_idx < topo_map->node_num; node_idx++) { + node = &topo_map->topo_infos[node_idx]; + + for (dev_idx = 0; dev_idx < DEV_NUM; dev_idx++) { + for (die_idx = 0; die_idx < IODIE_NUM; die_idx++) { + if (!is_eid_valid(node->devs[dev_idx].fe[die_idx].primary_eid)) + continue; + + has_any_primary_eid = true; + (void)memcpy(&eid_info.eid, + node->devs[dev_idx].fe[die_idx].primary_eid, + sizeof(union ubcore_eid)); + + dev = ubcore_get_device_by_eid(&eid_info.eid, + UBCORE_TRANSPORT_UB); + if (dev == NULL) { + ubcore_log_err("primary dev not exist, node %d dev %d die %d, eid: " EID_FMT "\n", + node_idx, dev_idx, die_idx, + EID_RAW_ARGS(node->devs[dev_idx].fe[die_idx].primary_eid)); + return -1; + } + + ret = ubcore_get_eid_index(dev, &eid_info.eid, + &eid_info.eid_index); + if (ret != 0) { + ubcore_log_err("Failed to get eid index\n"); + return ret; + } + + ret = ubcore_call_cm_eid_ops(dev, &eid_info, + UBCORE_MGMT_EVENT_EID_ADD); + if (ret != 0) { + ubcore_log_err("Failed to call cm eid ops\n"); + return ret; + } + + ubcore_log_info("Created jetty rsrc: node %d dev %d primary die %d, eid: " EID_FMT ", idx: %d\n", + node_idx, dev_idx, die_idx, + EID_RAW_ARGS(node->devs[dev_idx].fe[die_idx].primary_eid), + eid_info.eid_index); + } + } + } + + return has_any_primary_eid ? 0 : -1; } static int ubcore_cmd_set_topo(struct ubcore_global_file *file, @@ -250,7 +256,7 @@ static int ubcore_cmd_set_topo(struct ubcore_global_file *file, ubcore_log_err("Failed to create topo map\n"); return -ENOMEM; } - if (!is_bonding_and_primary_eid_valid(topo_map)) { + if (!is_aggr_and_primary_eid_valid(topo_map)) { ubcore_delete_global_topo_map(); ubcore_log_err("Invalid bonding or primary eid\n"); return -EINVAL; diff --git a/include/ub/urma/ubcore_types.h b/include/ub/urma/ubcore_types.h index 8aa1306ee321..21ecb6914b70 100644 --- a/include/ub/urma/ubcore_types.h +++ b/include/ub/urma/ubcore_types.h @@ -81,7 +81,7 @@ ((ptr) == NULL ? ERR_PTR(-(err)) : (ptr)) #define UBCORE_MAX_DSCP_NUM (64) -#define UBCORE_MAX_ROUTE_NUM 16 +#define UBCORE_MAX_ROUTE_NUM 16 enum ubcore_transport_type { UBCORE_TRANSPORT_INVALID = -1, -- Gitee