1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
#include "stdafx.h"
#include "ib_low.h"
namespace NNetliba {
static bool EnableROCEFlag = false;
void EnableROCE(bool f) {
EnableROCEFlag = f;
}
#if defined _linux_ && !defined CATBOOST_OPENSOURCE
static TMutex IBPortMutex;
static TIntrusivePtr<TIBPort> IBPort;
static bool IBWasInitialized;
TIntrusivePtr<TIBPort> GetIBDevice() {
TGuard<TMutex> gg(IBPortMutex);
if (IBWasInitialized) {
return IBPort;
}
IBWasInitialized = true;
try {
int rv = ibv_fork_init();
if (rv != 0) {
//printf("ibv_fork_init() failed");
return nullptr;
}
} catch (...) {
//we can not load ib interface, so no ib
return nullptr;
}
TIntrusivePtr<TIBContext> ctx;
TIntrusivePtr<TIBPort> resPort;
int numDevices;
ibv_device** deviceList = ibv_get_device_list(&numDevices);
//for (int i = 0; i < numDevices; ++i) {
// ibv_device *dev = deviceList[i];
// printf("Dev %d\n", i);
// printf("name:%s\ndev_name:%s\ndev_path:%s\nibdev_path:%s\n",
// dev->name,
// dev->dev_name,
// dev->dev_path,
// dev->ibdev_path);
// printf("get_device_name(): %s\n", ibv_get_device_name(dev));
// ui64 devGuid = ibv_get_device_guid(dev);
// printf("ibv_get_device_guid: %" PRIx64 "\n", devGuid);
// printf("node type: %s\n", ibv_node_type_str(dev->node_type));
// printf("\n");
//}
if (numDevices == 1) {
ctx = new TIBContext(deviceList[0]);
TIBContext::TLock ibContext(ctx);
ibv_device_attr devAttrs;
CHECK_Z(ibv_query_device(ibContext.GetContext(), &devAttrs));
for (int port = 1; port <= devAttrs.phys_port_cnt; ++port) {
ibv_port_attr portAttrs;
CHECK_Z(ibv_query_port(ibContext.GetContext(), port, &portAttrs));
//ibv_gid myAddress; // ipv6 address of this port;
//CHECK_Z(ibv_query_gid(ibContext.GetContext(), port, 0, &myAddress));
//{
// ibv_gid p = myAddress;
// for (int k = 0; k < 4; ++k) {
// DoSwap(p.raw[k], p.raw[7 - k]);
// DoSwap(p.raw[8 + k], p.raw[15 - k]);
// }
// printf("Port %d, address %" PRIx64 ":%" PRIx64 "\n",
// port,
// p.global.subnet_prefix,
// p.global.interface_id);
//}
// skip ROCE if flag is not set
if (portAttrs.lid == 0 && EnableROCEFlag == false) {
continue;
}
// bind to first active port
if (portAttrs.state == IBV_PORT_ACTIVE) {
resPort = new TIBPort(ctx, port);
break;
}
}
} else {
//printf("%d IB devices found, fail\n", numDevices);
ctx = nullptr;
}
ibv_free_device_list(deviceList);
IBPort = resPort;
return IBPort;
}
void MakeAH(ibv_ah_attr* res, TPtrArg<TIBPort> port, const TUdpAddress& remoteAddr, const TUdpAddress& localAddr, int serviceLevel) {
ibv_gid localGid, remoteGid;
localGid.global.subnet_prefix = localAddr.Network;
localGid.global.interface_id = localAddr.Interface;
remoteGid.global.subnet_prefix = remoteAddr.Network;
remoteGid.global.interface_id = remoteAddr.Interface;
Zero(*res);
res->is_global = 1;
res->port_num = port->GetPort();
res->sl = serviceLevel;
res->grh.dgid = remoteGid;
//res->grh.flow_label = 0;
res->grh.sgid_index = port->GetGIDIndex(localGid);
res->grh.hop_limit = 7;
//res->grh.traffic_class = 0;
}
#endif
}
|
