DPDK  17.11.0
rte_mempool.h
Go to the documentation of this file.
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5  * Copyright(c) 2016 6WIND S.A.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in
16  * the documentation and/or other materials provided with the
17  * distribution.
18  * * Neither the name of Intel Corporation nor the names of its
19  * contributors may be used to endorse or promote products derived
20  * from this software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #ifndef _RTE_MEMPOOL_H_
36 #define _RTE_MEMPOOL_H_
37 
65 #include <stdio.h>
66 #include <stdlib.h>
67 #include <stdint.h>
68 #include <errno.h>
69 #include <inttypes.h>
70 #include <sys/queue.h>
71 
72 #include <rte_spinlock.h>
73 #include <rte_log.h>
74 #include <rte_debug.h>
75 #include <rte_lcore.h>
76 #include <rte_memory.h>
77 #include <rte_branch_prediction.h>
78 #include <rte_ring.h>
79 #include <rte_memcpy.h>
80 #include <rte_common.h>
81 
82 #ifdef __cplusplus
83 extern "C" {
84 #endif
85 
86 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
87 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
88 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
90 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
91 
94 struct rte_mempool_debug_stats {
95  uint64_t put_bulk;
96  uint64_t put_objs;
97  uint64_t get_success_bulk;
98  uint64_t get_success_objs;
99  uint64_t get_fail_bulk;
100  uint64_t get_fail_objs;
102 #endif
103 
108  uint32_t size;
109  uint32_t flushthresh;
110  uint32_t len;
111  /*
112  * Cache is allocated to this size to allow it to overflow in certain
113  * cases to avoid needless emptying of cache.
114  */
115  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
117 
122  uint32_t elt_size;
123  uint32_t header_size;
124  uint32_t trailer_size;
125  uint32_t total_size;
127 };
128 
130 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
131  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
132 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
133 
134 /* "MP_<name>" */
135 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
136 
137 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
138 
140 #define MEMPOOL_PG_NUM_DEFAULT 1
141 
142 #ifndef RTE_MEMPOOL_ALIGN
143 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
144 #endif
145 
146 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
147 
158  STAILQ_ENTRY(rte_mempool_objhdr) next;
159  struct rte_mempool *mp;
161  union {
164  };
165 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
166  uint64_t cookie;
167 #endif
168 };
169 
173 STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
174 
175 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
176 
183 struct rte_mempool_objtlr {
184  uint64_t cookie;
185 };
186 
187 #endif
188 
192 STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
193 
198  void *opaque);
199 
207  STAILQ_ENTRY(rte_mempool_memhdr) next;
208  struct rte_mempool *mp;
209  void *addr;
211  union {
214  };
215  size_t len;
217  void *opaque;
218 };
219 
223 struct rte_mempool {
224  /*
225  * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI
226  * compatibility requirements, it could be changed to
227  * RTE_MEMPOOL_NAMESIZE next time the ABI changes
228  */
229  char name[RTE_MEMZONE_NAMESIZE];
231  union {
232  void *pool_data;
233  uint64_t pool_id;
234  };
235  void *pool_config;
236  const struct rte_memzone *mz;
237  unsigned int flags;
238  int socket_id;
239  uint32_t size;
240  uint32_t cache_size;
243  uint32_t elt_size;
244  uint32_t header_size;
245  uint32_t trailer_size;
247  unsigned private_data_size;
255  int32_t ops_index;
256 
259  uint32_t populated_size;
260  struct rte_mempool_objhdr_list elt_list;
261  uint32_t nb_mem_chunks;
262  struct rte_mempool_memhdr_list mem_list;
264 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
265 
266  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
267 #endif
269 
270 #define MEMPOOL_F_NO_SPREAD 0x0001
271 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
272 #define MEMPOOL_F_SP_PUT 0x0004
273 #define MEMPOOL_F_SC_GET 0x0008
274 #define MEMPOOL_F_POOL_CREATED 0x0010
275 #define MEMPOOL_F_NO_PHYS_CONTIG 0x0020
281 #define MEMPOOL_F_CAPA_PHYS_CONTIG 0x0040
282 
293 #define MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS 0x0080
294 
305 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
306 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
307  unsigned __lcore_id = rte_lcore_id(); \
308  if (__lcore_id < RTE_MAX_LCORE) { \
309  mp->stats[__lcore_id].name##_objs += n; \
310  mp->stats[__lcore_id].name##_bulk += 1; \
311  } \
312  } while(0)
313 #else
314 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
315 #endif
316 
325 #define MEMPOOL_HEADER_SIZE(mp, cs) \
326  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
327  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
328 
329 /* return the header of a mempool object (internal) */
330 static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
331 {
332  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
333  sizeof(struct rte_mempool_objhdr));
334 }
335 
345 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
346 {
347  struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
348  return hdr->mp;
349 }
350 
351 /* return the trailer of a mempool object (internal) */
352 static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
353 {
354  struct rte_mempool *mp = rte_mempool_from_obj(obj);
355  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
356 }
357 
372 void rte_mempool_check_cookies(const struct rte_mempool *mp,
373  void * const *obj_table_const, unsigned n, int free);
374 
375 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
376 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
377  rte_mempool_check_cookies(mp, obj_table_const, n, free)
378 #else
379 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
380 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
381 
382 #define RTE_MEMPOOL_OPS_NAMESIZE 32
394 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
395 
399 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
400 
404 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
405  void * const *obj_table, unsigned int n);
406 
410 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
411  void **obj_table, unsigned int n);
412 
416 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
417 
421 typedef int (*rte_mempool_get_capabilities_t)(const struct rte_mempool *mp,
422  unsigned int *flags);
423 
428 (const struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len);
429 
447 
448 #define RTE_MEMPOOL_MAX_OPS_IDX 16
459 struct rte_mempool_ops_table {
461  uint32_t num_ops;
467 
470 
480 static inline struct rte_mempool_ops *
481 rte_mempool_get_ops(int ops_index)
482 {
483  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
484 
485  return &rte_mempool_ops_table.ops[ops_index];
486 }
487 
497 int
498 rte_mempool_ops_alloc(struct rte_mempool *mp);
499 
513 static inline int
514 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
515  void **obj_table, unsigned n)
516 {
517  struct rte_mempool_ops *ops;
518 
519  ops = rte_mempool_get_ops(mp->ops_index);
520  return ops->dequeue(mp, obj_table, n);
521 }
522 
536 static inline int
537 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
538  unsigned n)
539 {
540  struct rte_mempool_ops *ops;
541 
542  ops = rte_mempool_get_ops(mp->ops_index);
543  return ops->enqueue(mp, obj_table, n);
544 }
545 
554 unsigned
555 rte_mempool_ops_get_count(const struct rte_mempool *mp);
556 
570 int
571 rte_mempool_ops_get_capabilities(const struct rte_mempool *mp,
572  unsigned int *flags);
590 int
591 rte_mempool_ops_register_memory_area(const struct rte_mempool *mp,
592  char *vaddr, rte_iova_t iova, size_t len);
593 
600 void
601 rte_mempool_ops_free(struct rte_mempool *mp);
602 
620 int
621 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
622  void *pool_config);
623 
634 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
635 
641 #define MEMPOOL_REGISTER_OPS(ops) \
642  void mp_hdlr_init_##ops(void); \
643  void __attribute__((constructor, used)) mp_hdlr_init_##ops(void)\
644  { \
645  rte_mempool_register_ops(&ops); \
646  }
647 
653 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
654  void *opaque, void *obj, unsigned obj_idx);
655 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
656 
662 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
663  void *opaque, struct rte_mempool_memhdr *memhdr,
664  unsigned mem_idx);
665 
672 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
673 
753 struct rte_mempool *
754 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
755  unsigned cache_size, unsigned private_data_size,
756  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
757  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
758  int socket_id, unsigned flags);
759 
818 struct rte_mempool *
819 rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
820  unsigned cache_size, unsigned private_data_size,
821  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
822  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
823  int socket_id, unsigned flags, void *vaddr,
824  const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift);
825 
860 struct rte_mempool *
861 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
862  unsigned cache_size, unsigned private_data_size,
863  int socket_id, unsigned flags);
874 void
875 rte_mempool_free(struct rte_mempool *mp);
876 
904 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
905  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
906  void *opaque);
907 
908 __rte_deprecated
909 int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr,
910  phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
911  void *opaque);
912 
939 int rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr,
940  const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift,
941  rte_mempool_memchunk_free_cb_t *free_cb, void *opaque);
942 
943 __rte_deprecated
944 int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr,
945  const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
946  rte_mempool_memchunk_free_cb_t *free_cb, void *opaque);
947 
972 int
973 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
974  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
975  void *opaque);
976 
991 
1005 int rte_mempool_populate_anon(struct rte_mempool *mp);
1006 
1022 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1023  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1024 
1040 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1041  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1042 
1051 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1052 
1067 struct rte_mempool_cache *
1068 rte_mempool_cache_create(uint32_t size, int socket_id);
1069 
1076 void
1078 
1087 static __rte_always_inline void
1089  struct rte_mempool *mp)
1090 {
1091  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1092  cache->len = 0;
1093 }
1094 
1105 static __rte_always_inline struct rte_mempool_cache *
1106 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1107 {
1108  if (mp->cache_size == 0)
1109  return NULL;
1110 
1111  if (lcore_id >= RTE_MAX_LCORE)
1112  return NULL;
1113 
1114  return &mp->local_cache[lcore_id];
1115 }
1116 
1129 static __rte_always_inline void
1130 __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1131  unsigned int n, struct rte_mempool_cache *cache)
1132 {
1133  void **cache_objs;
1134 
1135  /* increment stat now, adding in mempool always success */
1136  __MEMPOOL_STAT_ADD(mp, put, n);
1137 
1138  /* No cache provided or if put would overflow mem allocated for cache */
1139  if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1140  goto ring_enqueue;
1141 
1142  cache_objs = &cache->objs[cache->len];
1143 
1144  /*
1145  * The cache follows the following algorithm
1146  * 1. Add the objects to the cache
1147  * 2. Anything greater than the cache min value (if it crosses the
1148  * cache flush threshold) is flushed to the ring.
1149  */
1150 
1151  /* Add elements back into the cache */
1152  rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1153 
1154  cache->len += n;
1155 
1156  if (cache->len >= cache->flushthresh) {
1157  rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1158  cache->len - cache->size);
1159  cache->len = cache->size;
1160  }
1161 
1162  return;
1163 
1164 ring_enqueue:
1165 
1166  /* push remaining objects in ring */
1167 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1168  if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1169  rte_panic("cannot put objects in mempool\n");
1170 #else
1171  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1172 #endif
1173 }
1174 
1175 
1188 static __rte_always_inline void
1189 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1190  unsigned int n, struct rte_mempool_cache *cache)
1191 {
1192  __mempool_check_cookies(mp, obj_table, n, 0);
1193  __mempool_generic_put(mp, obj_table, n, cache);
1194 }
1195 
1210 static __rte_always_inline void
1211 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1212  unsigned int n)
1213 {
1214  struct rte_mempool_cache *cache;
1215  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1216  rte_mempool_generic_put(mp, obj_table, n, cache);
1217 }
1218 
1231 static __rte_always_inline void
1232 rte_mempool_put(struct rte_mempool *mp, void *obj)
1233 {
1234  rte_mempool_put_bulk(mp, &obj, 1);
1235 }
1236 
1251 static __rte_always_inline int
1252 __mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1253  unsigned int n, struct rte_mempool_cache *cache)
1254 {
1255  int ret;
1256  uint32_t index, len;
1257  void **cache_objs;
1258 
1259  /* No cache provided or cannot be satisfied from cache */
1260  if (unlikely(cache == NULL || n >= cache->size))
1261  goto ring_dequeue;
1262 
1263  cache_objs = cache->objs;
1264 
1265  /* Can this be satisfied from the cache? */
1266  if (cache->len < n) {
1267  /* No. Backfill the cache first, and then fill from it */
1268  uint32_t req = n + (cache->size - cache->len);
1269 
1270  /* How many do we require i.e. number to fill the cache + the request */
1271  ret = rte_mempool_ops_dequeue_bulk(mp,
1272  &cache->objs[cache->len], req);
1273  if (unlikely(ret < 0)) {
1274  /*
1275  * In the offchance that we are buffer constrained,
1276  * where we are not able to allocate cache + n, go to
1277  * the ring directly. If that fails, we are truly out of
1278  * buffers.
1279  */
1280  goto ring_dequeue;
1281  }
1282 
1283  cache->len += req;
1284  }
1285 
1286  /* Now fill in the response ... */
1287  for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1288  *obj_table = cache_objs[len];
1289 
1290  cache->len -= n;
1291 
1292  __MEMPOOL_STAT_ADD(mp, get_success, n);
1293 
1294  return 0;
1295 
1296 ring_dequeue:
1297 
1298  /* get remaining objects from ring */
1299  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1300 
1301  if (ret < 0)
1302  __MEMPOOL_STAT_ADD(mp, get_fail, n);
1303  else
1304  __MEMPOOL_STAT_ADD(mp, get_success, n);
1305 
1306  return ret;
1307 }
1308 
1329 static __rte_always_inline int
1330 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1331  unsigned int n, struct rte_mempool_cache *cache)
1332 {
1333  int ret;
1334  ret = __mempool_generic_get(mp, obj_table, n, cache);
1335  if (ret == 0)
1336  __mempool_check_cookies(mp, obj_table, n, 1);
1337  return ret;
1338 }
1339 
1362 static __rte_always_inline int
1363 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1364 {
1365  struct rte_mempool_cache *cache;
1366  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1367  return rte_mempool_generic_get(mp, obj_table, n, cache);
1368 }
1369 
1390 static __rte_always_inline int
1391 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1392 {
1393  return rte_mempool_get_bulk(mp, obj_p, 1);
1394 }
1395 
1408 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1409 
1422 unsigned int
1423 rte_mempool_in_use_count(const struct rte_mempool *mp);
1424 
1438 static inline int
1440 {
1441  return !!(rte_mempool_avail_count(mp) == mp->size);
1442 }
1443 
1457 static inline int
1459 {
1460  return !!(rte_mempool_avail_count(mp) == 0);
1461 }
1462 
1473 static inline rte_iova_t
1474 rte_mempool_virt2iova(const void *elt)
1475 {
1476  const struct rte_mempool_objhdr *hdr;
1477  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1478  sizeof(*hdr));
1479  return hdr->iova;
1480 }
1481 
1482 __rte_deprecated
1483 static inline phys_addr_t
1484 rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt)
1485 {
1486  return rte_mempool_virt2iova(elt);
1487 }
1488 
1499 void rte_mempool_audit(struct rte_mempool *mp);
1500 
1509 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1510 {
1511  return (char *)mp +
1513 }
1514 
1521 void rte_mempool_list_dump(FILE *f);
1522 
1535 struct rte_mempool *rte_mempool_lookup(const char *name);
1536 
1554 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1555  struct rte_mempool_objsz *sz);
1556 
1580 size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz,
1581  uint32_t pg_shift, unsigned int flags);
1582 
1611 ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num,
1612  size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num,
1613  uint32_t pg_shift, unsigned int flags);
1614 
1623 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1624  void *arg);
1625 
1626 #ifdef __cplusplus
1627 }
1628 #endif
1629 
1630 #endif /* _RTE_MEMPOOL_H_ */
uint64_t pool_id
Definition: rte_mempool.h:233
#define __rte_always_inline
Definition: rte_common.h:137
struct rte_mempool * rte_mempool_lookup(const char *name)
int(* rte_mempool_ops_register_memory_area_t)(const struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len)
Definition: rte_mempool.h:428
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:244
const struct rte_memzone * mz
Definition: rte_mempool.h:236
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1211
struct rte_mempool * mp
Definition: rte_mempool.h:208
rte_mempool_get_capabilities_t get_capabilities
Definition: rte_mempool.h:441
void rte_mempool_list_dump(FILE *f)
void() rte_mempool_memchunk_free_cb_t(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:197
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:433
unsigned int flags
Definition: rte_mempool.h:237
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1189
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:394
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1458
#define __rte_unused
Definition: rte_common.h:103
phys_addr_t physaddr
Definition: rte_mempool.h:163
phys_addr_t phys_addr
Definition: rte_mempool.h:213
struct rte_mbuf __rte_cache_aligned
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:216
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:432
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:410
#define MEMPOOL_HEADER_SIZE(mp, cs)
Definition: rte_mempool.h:325
uint32_t cache_size
Definition: rte_mempool.h:240
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num, size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, unsigned int flags)
uint32_t size
Definition: rte_mempool.h:239
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:149
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:125
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
int rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr, const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1330
int rte_mempool_populate_default(struct rte_mempool *mp)
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1088
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:382
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1391
STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
uint32_t nb_mem_chunks
Definition: rte_mempool.h:261
struct rte_mempool * mp
Definition: rte_mempool.h:159
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:465
rte_mempool_get_count get_count
Definition: rte_mempool.h:437
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1439
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:93
void * pool_config
Definition: rte_mempool.h:235
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1363
#define rte_panic(...)
Definition: rte_debug.h:79
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:416
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
void * objs[RTE_MEMPOOL_CACHE_MAX_SIZE *3]
Definition: rte_mempool.h:115
void * pool_data
Definition: rte_mempool.h:232
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
uint32_t elt_size
Definition: rte_mempool.h:243
void rte_mempool_audit(struct rte_mempool *mp)
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
unsigned private_data_size
Definition: rte_mempool.h:247
rte_spinlock_t sl
Definition: rte_mempool.h:460
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1106
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
void() rte_mempool_ctor_t(struct rte_mempool *, void *)
Definition: rte_mempool.h:672
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:257
uint32_t trailer_size
Definition: rte_mempool.h:245
#define RTE_STD_C11
Definition: rte_common.h:64
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:436
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:404
rte_mempool_ops_register_memory_area_t register_memory_area
Definition: rte_mempool.h:445
int(* rte_mempool_get_capabilities_t)(const struct rte_mempool *mp, unsigned int *flags)
Definition: rte_mempool.h:421
uint64_t rte_iova_t
Definition: rte_memory.h:106
uint32_t flushthresh
Definition: rte_mempool.h:109
void() rte_mempool_obj_cb_t(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:653
void() rte_mempool_mem_cb_t(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:662
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
uint64_t phys_addr_t
Definition: rte_memory.h:97
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:154
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:345
uint32_t header_size
Definition: rte_mempool.h:123
int32_t ops_index
Definition: rte_mempool.h:255
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:259
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1474
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1232
size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift, unsigned int flags)
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1509
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:229
uint32_t trailer_size
Definition: rte_mempool.h:124
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:448
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:399
rte_mempool_free_t free
Definition: rte_mempool.h:434
struct rte_mempool * rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags, void *vaddr, const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift)
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:435
#define RTE_MEMZONE_NAMESIZE
Definition: rte_memzone.h:78