A Discrete-Event Network Simulator
API
Loading...
Searching...
No Matches
dpdk-net-device.cc
Go to the documentation of this file.
1/*
2 * Copyright (c) 2019 NITK Surathkal
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 *
6 * Author: Harsh Patel <thadodaharsh10@gmail.com>
7 * Hrishikesh Hiraskar <hrishihiraskar@gmail.com>
8 * Mohit P. Tahiliani <tahiliani@nitk.edu.in>
9 */
10
11#include "dpdk-net-device.h"
12
13#include "ns3/log.h"
14#include "ns3/net-device-queue-interface.h"
15#include "ns3/simulator.h"
16#include "ns3/uinteger.h"
17
18#include <mutex>
19#include <poll.h>
20#include <rte_common.h>
21#include <rte_cycles.h>
22#include <rte_eal.h>
23#include <rte_ethdev.h>
24#include <rte_malloc.h>
25#include <rte_mbuf.h>
26#include <rte_mempool.h>
27#include <rte_port.h>
28#include <sys/ioctl.h>
29#include <sys/mman.h>
30#include <sys/signal.h>
31#include <unistd.h>
32
33namespace ns3
34{
35
36NS_LOG_COMPONENT_DEFINE("DpdkNetDevice");
37
38NS_OBJECT_ENSURE_REGISTERED(DpdkNetDevice);
39
40volatile bool DpdkNetDevice::m_forceQuit = false;
41
42TypeId
44{
45 static TypeId tid =
46 TypeId("ns3::DpdkNetDevice")
48 .SetGroupName("FdNetDevice")
49 .AddConstructor<DpdkNetDevice>()
50 .AddAttribute("TxTimeout",
51 "The time to wait before transmitting burst from Tx buffer.",
55 .AddAttribute("MaxRxBurst",
56 "Size of Rx Burst.",
57 UintegerValue(64),
60 .AddAttribute("MaxTxBurst",
61 "Size of Tx Burst.",
62 UintegerValue(64),
65 .AddAttribute("MempoolCacheSize",
66 "Size of mempool cache.",
67 UintegerValue(256),
70 .AddAttribute("NbRxDesc",
71 "Number of Rx descriptors.",
72 UintegerValue(1024),
75 .AddAttribute("NbTxDesc",
76 "Number of Tx descriptors.",
77 UintegerValue(1024),
80 return tid;
81}
82
84 : m_mempool(nullptr)
85{
86 NS_LOG_FUNCTION(this);
87}
88
90{
91 NS_LOG_FUNCTION(this);
93 m_forceQuit = true;
94
95 rte_eal_wait_lcore(1);
96 rte_eth_dev_stop(m_portId);
97 rte_eth_dev_close(m_portId);
98}
99
100void
101DpdkNetDevice::SetDeviceName(std::string deviceName)
102{
103 NS_LOG_FUNCTION(this);
104
105 m_deviceName = deviceName;
106}
107
108void
110{
111 NS_LOG_FUNCTION(this);
112
113#define CHECK_INTERVAL 100 /* 100ms */
114#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
115
116 uint8_t printFlag = 0;
117 struct rte_eth_link link;
118
119 for (uint8_t count = 0; count <= MAX_CHECK_TIME; count++)
120 {
121 uint8_t allPortsUp = 1;
122
123 if (m_forceQuit)
124 {
125 return;
126 }
127 if ((1 << m_portId) == 0)
128 {
129 continue;
130 }
131 memset(&link, 0, sizeof(link));
132 rte_eth_link_get(m_portId, &link);
133 /* print link status if flag set */
134 if (printFlag == 1)
135 {
136 if (!link.link_status)
137 {
138 NS_LOG_INFO("Port " << +m_portId << " Link Down");
139 }
140
141 continue;
142 }
143 /* clear allPortsUp flag if any link down */
144 if (link.link_status == ETH_LINK_DOWN)
145 {
146 allPortsUp = 0;
147 break;
148 }
149
150 /* after finally printing all link status, get out */
151 if (printFlag == 1)
152 {
153 break;
154 }
155
156 if (allPortsUp == 0)
157 {
158 fflush(stdout);
159 rte_delay_ms(CHECK_INTERVAL);
160 }
161
162 /* set the printFlag if all ports up or timeout */
163 if (allPortsUp == 1 || count == (MAX_CHECK_TIME - 1))
164 {
165 printFlag = 1;
166 }
167 }
168}
169
170void
172{
173 if (signum == SIGINT || signum == SIGTERM)
174 {
175 NS_LOG_INFO("Signal " << signum << " received, preparing to exit...");
176 m_forceQuit = true;
177 }
178}
179
180void
182{
183 int queueId = 0;
184 rte_eth_tx_buffer_flush(m_portId, queueId, m_txBuffer);
185}
186
187void
189{
190 int queueId = 0;
191 m_rxBuffer->length = rte_eth_rx_burst(m_portId, queueId, m_rxBuffer->pkts, m_maxRxPktBurst);
192
193 for (uint16_t i = 0; i < m_rxBuffer->length; i++)
194 {
195 struct rte_mbuf* pkt = nullptr;
196 pkt = m_rxBuffer->pkts[i];
197
198 if (!pkt)
199 {
200 continue;
201 }
202
203 uint8_t* buf = rte_pktmbuf_mtod(pkt, uint8_t*);
204 size_t length = pkt->data_len;
205 FdNetDevice::ReceiveCallback(buf, length);
206 }
207
208 m_rxBuffer->length = 0;
209}
210
211int
213{
214 DpdkNetDevice* dpdkNetDevice = (DpdkNetDevice*)arg;
215 unsigned lcoreId;
216 lcoreId = rte_lcore_id();
217 if (lcoreId != 1)
218 {
219 return 0;
220 }
221
222 while (!m_forceQuit)
223 {
224 dpdkNetDevice->HandleRx();
225 }
226
227 return 0;
228}
229
230bool
232{
233 // Refer https://mails.dpdk.org/archives/users/2018-December/003822.html
234 return true;
235}
236
237void
238DpdkNetDevice::InitDpdk(int argc, char** argv, std::string dpdkDriver)
239{
240 NS_LOG_FUNCTION(this << argc << argv);
241
242 NS_LOG_INFO("Binding device to DPDK");
243 std::string command;
244 command.append("dpdk-devbind.py --force ");
245 command.append("--bind=");
246 command.append(dpdkDriver);
247 command.append(" ");
248 command.append(m_deviceName);
249 NS_LOG_INFO("Executing: " << command);
250 if (system(command.c_str()))
251 {
252 rte_exit(EXIT_FAILURE, "Execution failed - bye\n");
253 }
254
255 // wait for the device to bind to Dpdk
256 sleep(5); /* 5 seconds */
257
258 NS_LOG_INFO("Initialize DPDK EAL");
259 int ret = rte_eal_init(argc, argv);
260 if (ret < 0)
261 {
262 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
263 }
264
265 m_forceQuit = false;
266 signal(SIGINT, SignalHandler);
267 signal(SIGTERM, SignalHandler);
268
269 unsigned nbPorts = rte_eth_dev_count_avail();
270 if (nbPorts == 0)
271 {
272 rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
273 }
274
275 NS_LOG_INFO("Get port id of the device");
276 if (rte_eth_dev_get_port_by_name(m_deviceName.c_str(), &m_portId) != 0)
277 {
278 rte_exit(EXIT_FAILURE, "Cannot get port id - bye\n");
279 }
280
281 // Set number of logical cores to 2
282 unsigned int nbLcores = 2;
283
284 unsigned int nbMbufs = RTE_MAX(nbPorts * (m_nbRxDesc + m_nbTxDesc + m_maxRxPktBurst +
286 8192U);
287
288 NS_LOG_INFO("Create the mbuf pool");
289 m_mempool = rte_pktmbuf_pool_create("mbuf_pool",
290 nbMbufs,
292 0,
293 RTE_MBUF_DEFAULT_BUF_SIZE,
294 rte_socket_id());
295
296 if (!m_mempool)
297 {
298 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n");
299 }
300
301 NS_LOG_INFO("Initialize port");
302 static struct rte_eth_conf portConf = {};
303 portConf.rxmode = {};
304 portConf.rxmode.split_hdr_size = 0;
305 portConf.txmode = {};
306 portConf.txmode.mq_mode = ETH_MQ_TX_NONE;
307
308 struct rte_eth_rxconf reqConf;
309 struct rte_eth_txconf txqConf;
310 struct rte_eth_conf localPortConf = portConf;
311 struct rte_eth_dev_info devInfo;
312
313 fflush(stdout);
314 rte_eth_dev_info_get(m_portId, &devInfo);
315 if (devInfo.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
316 {
317 localPortConf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
318 }
319 ret = rte_eth_dev_configure(m_portId, 1, 1, &localPortConf);
320 if (ret < 0)
321 {
322 rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", ret, m_portId);
323 }
324
325 ret = rte_eth_dev_adjust_nb_rx_tx_desc(m_portId, &m_nbRxDesc, &m_nbTxDesc);
326 if (ret < 0)
327 {
328 rte_exit(EXIT_FAILURE,
329 "Cannot adjust number of descriptors: err=%d, port=%u\n",
330 ret,
331 m_portId);
332 }
333
334 NS_LOG_INFO("Initialize one Rx queue");
335 fflush(stdout);
336 reqConf = devInfo.default_rxconf;
337 reqConf.offloads = localPortConf.rxmode.offloads;
338 ret = rte_eth_rx_queue_setup(m_portId,
339 0,
341 rte_eth_dev_socket_id(m_portId),
342 &reqConf,
343 m_mempool);
344 if (ret < 0)
345 {
346 rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n", ret, m_portId);
347 }
348
349 NS_LOG_INFO("Initialize one Tx queue per port");
350 fflush(stdout);
351 txqConf = devInfo.default_txconf;
352 txqConf.offloads = localPortConf.txmode.offloads;
353 ret =
354 rte_eth_tx_queue_setup(m_portId, 0, m_nbTxDesc, rte_eth_dev_socket_id(m_portId), &txqConf);
355 if (ret < 0)
356 {
357 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n", ret, m_portId);
358 }
359
360 NS_LOG_INFO("Initialize Tx buffers");
361 m_txBuffer = (rte_eth_dev_tx_buffer*)rte_zmalloc_socket("tx_buffer",
362 RTE_ETH_TX_BUFFER_SIZE(m_maxTxPktBurst),
363 0,
364 rte_eth_dev_socket_id(m_portId));
365 NS_LOG_INFO("Initialize Rx buffers");
366 m_rxBuffer = (rte_eth_dev_tx_buffer*)rte_zmalloc_socket("rx_buffer",
367 RTE_ETH_TX_BUFFER_SIZE(m_maxRxPktBurst),
368 0,
369 rte_eth_dev_socket_id(m_portId));
370 if (!m_txBuffer || !m_rxBuffer)
371 {
372 rte_exit(EXIT_FAILURE, "Cannot allocate buffer for rx/tx on port %u\n", m_portId);
373 }
374
375 rte_eth_tx_buffer_init(m_txBuffer, m_maxTxPktBurst);
376 rte_eth_tx_buffer_init(m_rxBuffer, m_maxRxPktBurst);
377
378 NS_LOG_INFO("Start the device");
379 ret = rte_eth_dev_start(m_portId);
380 if (ret < 0)
381 {
382 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", ret, m_portId);
383 }
384
385 rte_eth_promiscuous_enable(m_portId);
386
388
389 NS_LOG_INFO("Launching core threads");
390 rte_eal_mp_remote_launch(LaunchCore, this, CALL_MASTER);
391}
392
393uint8_t*
395{
396 struct rte_mbuf* pkt = rte_pktmbuf_alloc(m_mempool);
397 if (!pkt)
398 {
399 return nullptr;
400 }
401 uint8_t* buf = rte_pktmbuf_mtod(pkt, uint8_t*);
402 return buf;
403}
404
405void
407{
408 struct rte_mbuf* pkt;
409
410 if (!buf)
411 {
412 return;
413 }
414 pkt = (struct rte_mbuf*)RTE_PTR_SUB(buf, sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);
415
416 rte_pktmbuf_free(pkt);
417}
418
419ssize_t
420DpdkNetDevice::Write(uint8_t* buffer, size_t length)
421{
422 struct rte_mbuf** pkt = new struct rte_mbuf*[1];
423 int queueId = 0;
424
425 if (!buffer || m_txBuffer->length == m_maxTxPktBurst)
426 {
427 NS_LOG_ERROR("Error allocating mbuf" << buffer);
428 return -1;
429 }
430
431 pkt[0] = (struct rte_mbuf*)RTE_PTR_SUB(buffer, sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);
432
433 pkt[0]->pkt_len = length;
434 pkt[0]->data_len = length;
435 rte_eth_tx_buffer(m_portId, queueId, m_txBuffer, pkt[0]);
436
437 if (m_txBuffer->length == 1)
438 {
439 // If this is a first packet in buffer, schedule a tx.
442 }
443
444 return length;
445}
446
447void
449{
450 std::unique_lock lock{m_pendingReadMutex};
451
452 while (!m_pendingQueue.empty())
453 {
454 std::pair<uint8_t*, ssize_t> next = m_pendingQueue.front();
455 m_pendingQueue.pop();
456
457 FreeBuffer(next.first);
458 }
459}
460
461} // namespace ns3
a NetDevice to read/write network traffic from/into a Dpdk enabled port.
static int LaunchCore(void *arg)
A function to handle rx & tx operations.
uint32_t m_maxRxPktBurst
Size of Rx burst.
void InitDpdk(int argc, char **argv, std::string dpdkDriver)
Initialize Dpdk.
void SetDeviceName(std::string deviceName)
Set device name.
void HandleTx()
Transmit packets in burst from the tx_buffer to the nic.
static void SignalHandler(int signum)
A signal handler for SIGINT and SIGTERM signals.
void FreeBuffer(uint8_t *buf) override
Free the given packet buffer.
struct rte_eth_dev_tx_buffer * m_txBuffer
Buffer to handle burst transmission.
struct rte_eth_dev_tx_buffer * m_rxBuffer
Buffer to handle burst reception.
uint32_t m_maxTxPktBurst
Size of Tx burst.
static TypeId GetTypeId()
Get the type ID.
EventId m_txEvent
Event for stale packet transmission.
ssize_t Write(uint8_t *buffer, size_t length) override
Write packet data to device.
std::string m_deviceName
The device name;.
~DpdkNetDevice() override
Destructor for the DpdkNetDevice.
static volatile bool m_forceQuit
Condition variable for Dpdk to stop.
bool IsLinkUp() const override
Check the status of the link.
uint16_t m_nbTxDesc
Number of Tx descriptors.
uint16_t m_nbRxDesc
Number of Rx descriptors.
uint8_t * AllocateBuffer(size_t len) override
Allocate packet buffer.
struct rte_mempool * m_mempool
Packet memory pool.
void CheckAllPortsLinkStatus()
Check the link status of all ports in up to 9s and print them finally.
uint16_t m_portId
The port number of the device to be used.
void DoFinishStoppingDevice() override
Complete additional actions, if any, to tear down the device.
DpdkNetDevice()
Constructor for the DpdkNetDevice.
void HandleRx()
Receive packets in burst from the nic to the rx_buffer.
Time m_txTimeout
The time to wait before transmitting burst from Tx buffer.
uint32_t m_mempoolCacheSize
Mempool cache size.
a NetDevice to read/write network traffic from/into a file descriptor.
std::mutex m_pendingReadMutex
Mutex to increase pending read counter.
std::queue< std::pair< uint8_t *, ssize_t > > m_pendingQueue
Number of packets that were received and scheduled for read but not yet read.
Callback< bool, Ptr< NetDevice >, Ptr< const Packet >, uint16_t, const Address & > ReceiveCallback
Definition net-device.h:311
static EventId Schedule(const Time &delay, FUNC f, Ts &&... args)
Schedule an event to expire after delay.
Definition simulator.h:560
static void Cancel(const EventId &id)
Set the cancel bit on this event: the event's associated function will not be invoked when it expires...
Definition simulator.cc:274
a unique identifier for an interface.
Definition type-id.h:48
TypeId SetParent(TypeId tid)
Set the parent TypeId.
Definition type-id.cc:1001
Hold an unsigned integer type.
Definition uinteger.h:34
#define MAX_CHECK_TIME
#define CHECK_INTERVAL
#define NS_LOG_ERROR(msg)
Use NS_LOG to output a message of level LOG_ERROR.
Definition log.h:243
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition log.h:191
#define NS_LOG_FUNCTION(parameters)
If log level LOG_FUNCTION is enabled, this macro will output all input parameters separated by ",...
#define NS_LOG_INFO(msg)
Use NS_LOG to output a message of level LOG_INFO.
Definition log.h:264
#define NS_OBJECT_ENSURE_REGISTERED(type)
Register an Object subclass with the TypeId system.
Definition object-base.h:35
Time MicroSeconds(uint64_t value)
Construct a Time in the indicated unit.
Definition nstime.h:1332
Every class exported by the ns3 library is enclosed in the ns3 namespace.
Ptr< const AttributeChecker > MakeUintegerChecker()
Definition uinteger.h:85
Ptr< const AttributeAccessor > MakeTimeAccessor(T1 a1)
Definition nstime.h:1396
Ptr< const AttributeAccessor > MakeUintegerAccessor(T1 a1)
Definition uinteger.h:35
Ptr< const AttributeChecker > MakeTimeChecker()
Helper to make an unbounded Time checker.
Definition nstime.h:1416