11 #ifndef _RTE_RING_ELEM_H_
12 #define _RTE_RING_ELEM_H_
105 unsigned int count,
int socket_id,
unsigned int flags);
108 __rte_ring_enqueue_elems_32(
struct rte_ring *r,
const uint32_t
size,
109 uint32_t idx,
const void *obj_table, uint32_t n)
112 uint32_t *ring = (uint32_t *)&r[1];
113 const uint32_t *obj = (
const uint32_t *)obj_table;
115 for (i = 0; i < (n & ~0x7); i += 8, idx += 8) {
117 ring[idx + 1] = obj[i + 1];
118 ring[idx + 2] = obj[i + 2];
119 ring[idx + 3] = obj[i + 3];
120 ring[idx + 4] = obj[i + 4];
121 ring[idx + 5] = obj[i + 5];
122 ring[idx + 6] = obj[i + 6];
123 ring[idx + 7] = obj[i + 7];
127 ring[idx++] = obj[i++];
129 ring[idx++] = obj[i++];
131 ring[idx++] = obj[i++];
133 ring[idx++] = obj[i++];
135 ring[idx++] = obj[i++];
137 ring[idx++] = obj[i++];
139 ring[idx++] = obj[i++];
142 for (i = 0; idx <
size; i++, idx++)
145 for (idx = 0; i < n; i++, idx++)
151 __rte_ring_enqueue_elems_64(
struct rte_ring *r, uint32_t prod_head,
152 const void *obj_table, uint32_t n)
156 uint32_t idx = prod_head & r->
mask;
157 uint64_t *ring = (uint64_t *)&r[1];
158 const unaligned_uint64_t *obj = (
const unaligned_uint64_t *)obj_table;
160 for (i = 0; i < (n & ~0x3); i += 4, idx += 4) {
162 ring[idx + 1] = obj[i + 1];
163 ring[idx + 2] = obj[i + 2];
164 ring[idx + 3] = obj[i + 3];
168 ring[idx++] = obj[i++];
170 ring[idx++] = obj[i++];
172 ring[idx++] = obj[i++];
175 for (i = 0; idx <
size; i++, idx++)
178 for (idx = 0; i < n; i++, idx++)
184 __rte_ring_enqueue_elems_128(
struct rte_ring *r, uint32_t prod_head,
185 const void *obj_table, uint32_t n)
189 uint32_t idx = prod_head & r->
mask;
190 rte_int128_t *ring = (rte_int128_t *)&r[1];
191 const rte_int128_t *obj = (
const rte_int128_t *)obj_table;
193 for (i = 0; i < (n & ~0x1); i += 2, idx += 2)
194 memcpy((
void *)(ring + idx),
195 (
const void *)(obj + i), 32);
198 memcpy((
void *)(ring + idx),
199 (
const void *)(obj + i), 16);
202 for (i = 0; idx <
size; i++, idx++)
203 memcpy((
void *)(ring + idx),
204 (
const void *)(obj + i), 16);
206 for (idx = 0; i < n; i++, idx++)
207 memcpy((
void *)(ring + idx),
208 (
const void *)(obj + i), 16);
217 __rte_ring_enqueue_elems(
struct rte_ring *r, uint32_t prod_head,
218 const void *obj_table, uint32_t esize, uint32_t num)
224 __rte_ring_enqueue_elems_64(r, prod_head, obj_table, num);
225 else if (esize == 16)
226 __rte_ring_enqueue_elems_128(r, prod_head, obj_table, num);
228 uint32_t idx, scale, nr_idx, nr_num, nr_size;
231 scale = esize /
sizeof(uint32_t);
232 nr_num = num * scale;
233 idx = prod_head & r->
mask;
234 nr_idx = idx * scale;
235 nr_size = r->
size * scale;
236 __rte_ring_enqueue_elems_32(r, nr_size, nr_idx,
242 __rte_ring_dequeue_elems_32(
struct rte_ring *r,
const uint32_t
size,
243 uint32_t idx,
void *obj_table, uint32_t n)
246 uint32_t *ring = (uint32_t *)&r[1];
247 uint32_t *obj = (uint32_t *)obj_table;
249 for (i = 0; i < (n & ~0x7); i += 8, idx += 8) {
251 obj[i + 1] = ring[idx + 1];
252 obj[i + 2] = ring[idx + 2];
253 obj[i + 3] = ring[idx + 3];
254 obj[i + 4] = ring[idx + 4];
255 obj[i + 5] = ring[idx + 5];
256 obj[i + 6] = ring[idx + 6];
257 obj[i + 7] = ring[idx + 7];
261 obj[i++] = ring[idx++];
263 obj[i++] = ring[idx++];
265 obj[i++] = ring[idx++];
267 obj[i++] = ring[idx++];
269 obj[i++] = ring[idx++];
271 obj[i++] = ring[idx++];
273 obj[i++] = ring[idx++];
276 for (i = 0; idx <
size; i++, idx++)
279 for (idx = 0; i < n; i++, idx++)
285 __rte_ring_dequeue_elems_64(
struct rte_ring *r, uint32_t prod_head,
286 void *obj_table, uint32_t n)
290 uint32_t idx = prod_head & r->
mask;
291 uint64_t *ring = (uint64_t *)&r[1];
292 unaligned_uint64_t *obj = (unaligned_uint64_t *)obj_table;
294 for (i = 0; i < (n & ~0x3); i += 4, idx += 4) {
296 obj[i + 1] = ring[idx + 1];
297 obj[i + 2] = ring[idx + 2];
298 obj[i + 3] = ring[idx + 3];
302 obj[i++] = ring[idx++];
304 obj[i++] = ring[idx++];
306 obj[i++] = ring[idx++];
309 for (i = 0; idx <
size; i++, idx++)
312 for (idx = 0; i < n; i++, idx++)
318 __rte_ring_dequeue_elems_128(
struct rte_ring *r, uint32_t prod_head,
319 void *obj_table, uint32_t n)
323 uint32_t idx = prod_head & r->
mask;
324 rte_int128_t *ring = (rte_int128_t *)&r[1];
325 rte_int128_t *obj = (rte_int128_t *)obj_table;
327 for (i = 0; i < (n & ~0x1); i += 2, idx += 2)
328 memcpy((
void *)(obj + i), (
void *)(ring + idx), 32);
331 memcpy((
void *)(obj + i), (
void *)(ring + idx), 16);
334 for (i = 0; idx <
size; i++, idx++)
335 memcpy((
void *)(obj + i), (
void *)(ring + idx), 16);
337 for (idx = 0; i < n; i++, idx++)
338 memcpy((
void *)(obj + i), (
void *)(ring + idx), 16);
347 __rte_ring_dequeue_elems(
struct rte_ring *r, uint32_t cons_head,
348 void *obj_table, uint32_t esize, uint32_t num)
354 __rte_ring_dequeue_elems_64(r, cons_head, obj_table, num);
355 else if (esize == 16)
356 __rte_ring_dequeue_elems_128(r, cons_head, obj_table, num);
358 uint32_t idx, scale, nr_idx, nr_num, nr_size;
361 scale = esize /
sizeof(uint32_t);
362 nr_num = num * scale;
363 idx = cons_head & r->
mask;
364 nr_idx = idx * scale;
365 nr_size = r->
size * scale;
366 __rte_ring_dequeue_elems_32(r, nr_size, nr_idx,
379 #ifdef RTE_USE_C11_MEM_MODEL
380 #include "rte_ring_c11_mem.h"
382 #include "rte_ring_generic.h"
410 __rte_ring_do_enqueue_elem(
struct rte_ring *r,
const void *obj_table,
411 unsigned int esize,
unsigned int n,
413 unsigned int *free_space)
415 uint32_t prod_head, prod_next;
416 uint32_t free_entries;
418 n = __rte_ring_move_prod_head(r, is_sp, n, behavior,
419 &prod_head, &prod_next, &free_entries);
423 __rte_ring_enqueue_elems(r, prod_head, obj_table, esize, n);
425 update_tail(&r->prod, prod_head, prod_next, is_sp, 1);
427 if (free_space != NULL)
428 *free_space = free_entries - n;
457 __rte_ring_do_dequeue_elem(
struct rte_ring *r,
void *obj_table,
458 unsigned int esize,
unsigned int n,
460 unsigned int *available)
462 uint32_t cons_head, cons_next;
465 n = __rte_ring_move_cons_head(r, (
int)is_sc, n, behavior,
466 &cons_head, &cons_next, &entries);
470 __rte_ring_dequeue_elems(r, cons_head, obj_table, esize, n);
472 update_tail(&r->cons, cons_head, cons_next, is_sc, 0);
475 if (available != NULL)
476 *available = entries - n;
504 unsigned int esize,
unsigned int n,
unsigned int *free_space)
506 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
533 unsigned int esize,
unsigned int n,
unsigned int *free_space)
535 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
539 #ifdef ALLOW_EXPERIMENTAL_API
569 unsigned int esize,
unsigned int n,
unsigned int *free_space)
578 #ifdef ALLOW_EXPERIMENTAL_API
579 case RTE_RING_SYNC_MT_RTS:
582 case RTE_RING_SYNC_MT_HTS:
590 if (free_space != NULL)
694 unsigned int esize,
unsigned int n,
unsigned int *available)
696 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
722 unsigned int esize,
unsigned int n,
unsigned int *available)
724 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
753 unsigned int esize,
unsigned int n,
unsigned int *available)
762 #ifdef ALLOW_EXPERIMENTAL_API
763 case RTE_RING_SYNC_MT_RTS:
766 case RTE_RING_SYNC_MT_HTS:
774 if (available != NULL)
881 unsigned int esize,
unsigned int n,
unsigned int *free_space)
883 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
910 unsigned int esize,
unsigned int n,
unsigned int *free_space)
912 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
941 unsigned int esize,
unsigned int n,
unsigned int *free_space)
950 #ifdef ALLOW_EXPERIMENTAL_API
951 case RTE_RING_SYNC_MT_RTS:
954 case RTE_RING_SYNC_MT_HTS:
962 if (free_space != NULL)
993 unsigned int esize,
unsigned int n,
unsigned int *available)
995 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
1022 unsigned int esize,
unsigned int n,
unsigned int *available)
1024 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
1053 unsigned int esize,
unsigned int n,
unsigned int *available)
1062 #ifdef ALLOW_EXPERIMENTAL_API
1063 case RTE_RING_SYNC_MT_RTS:
1066 case RTE_RING_SYNC_MT_HTS:
1074 if (available != NULL)
1079 #ifdef ALLOW_EXPERIMENTAL_API
#define __rte_always_inline
@ RTE_RING_QUEUE_VARIABLE
static __rte_always_inline int rte_ring_sp_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
static __rte_always_inline int rte_ring_mc_dequeue_elem(struct rte_ring *r, void *obj_p, unsigned int esize)
static __rte_always_inline unsigned int rte_ring_sc_dequeue_burst_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_dequeue_bulk_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_mc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_always_inline int rte_ring_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
static __rte_always_inline int rte_ring_sc_dequeue_elem(struct rte_ring *r, void *obj_p, unsigned int esize)
static __rte_always_inline int rte_ring_dequeue_elem(struct rte_ring *r, void *obj_p, unsigned int esize)
static __rte_always_inline unsigned int rte_ring_mc_dequeue_burst_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_always_inline int rte_ring_mp_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
static __rte_always_inline unsigned int rte_ring_sc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_mp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_always_inline unsigned int rte_ring_mp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
ssize_t rte_ring_get_memsize_elem(unsigned int esize, unsigned int count)
struct rte_ring * rte_ring_create_elem(const char *name, unsigned int esize, unsigned int count, int socket_id, unsigned int flags)
static __rte_always_inline unsigned int rte_ring_sp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_always_inline unsigned int rte_ring_dequeue_burst_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_always_inline unsigned int rte_ring_enqueue_burst_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_always_inline unsigned int rte_ring_sp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mp_hts_enqueue_burst_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mc_hts_dequeue_burst_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mc_hts_dequeue_bulk_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mp_hts_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mp_rts_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mc_rts_dequeue_bulk_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mc_rts_dequeue_burst_elem(struct rte_ring *r, void *obj_table, unsigned int esize, unsigned int n, unsigned int *available)
static __rte_experimental __rte_always_inline unsigned int rte_ring_mp_rts_enqueue_burst_elem(struct rte_ring *r, const void *obj_table, unsigned int esize, unsigned int n, unsigned int *free_space)
enum rte_ring_sync_type sync_type