.\" Automatically generated by Pandoc 3.8.2.1
.\"
.TH "MLX5DV_WR" "3" "2019\-02\-24" "mlx5" "mlx5 Programmer\(cqs Manual"
.SH NAME
mlx5dv_wr_set_dc_addr \- Attach a DC info to the last work request
.PP
mlx5dv_wr_raw_wqe \- Build a raw work request
.PP
mlx5dv_wr_memcpy \- Build a DMA memcpy work request
.SH SYNOPSIS
.IP
.EX
#include \f[B]<infiniband/mlx5dv.h>\f[R]

static \f[B]inline\f[R] void mlx5dv_wr_set_dc_addr(\f[B]struct\f[R] mlx5dv_qp_ex *mqp,
                                         \f[B]struct\f[R] ibv_ah *ah,
                                         uint32_t remote_dctn,
                                         uint64_t remote_dc_key);

static \f[B]inline\f[R] void mlx5dv_wr_set_dc_addr_stream(\f[B]struct\f[R] mlx5dv_qp_ex *mqp,
                        \f[B]struct\f[R] ibv_ah *ah,
                        uint32_t remote_dctn,
                        uint64_t remote_dc_key,
                        uint16_t stream_id);

\f[B]struct\f[R] mlx5dv_mr_interleaved {
    uint64_t        addr;
    uint32_t        bytes_count;
    uint32_t        bytes_skip;
    uint32_t        lkey;
};

static \f[B]inline\f[R] void mlx5dv_wr_mr_interleaved(\f[B]struct\f[R] mlx5dv_qp_ex *mqp,
                        \f[B]struct\f[R] mlx5dv_mkey *mkey,
                        uint32_t access_flags, \f[I]/* use enum ibv_access_flags */\f[R]
                        uint32_t repeat_count,
                        uint16_t num_interleaved,
                        \f[B]struct\f[R] mlx5dv_mr_interleaved *data);

static \f[B]inline\f[R] void mlx5dv_wr_mr_list(\f[B]struct\f[R] mlx5dv_qp_ex *mqp,
                      \f[B]struct\f[R] mlx5dv_mkey *mkey,
                      uint32_t access_flags, \f[I]/* use enum ibv_access_flags */\f[R]
                      uint16_t num_sges,
                      \f[B]struct\f[R] ibv_sge *sge);

static \f[B]inline\f[R] int mlx5dv_wr_raw_wqe(\f[B]struct\f[R] mlx5dv_qp_ex *mqp, const void *wqe);

static \f[B]inline\f[R] void mlx5dv_wr_memcpy(\f[B]struct\f[R] mlx5dv_qp_ex *mqp_ex,
                    uint32_t dest_lkey, uint64_t dest_addr,
                    uint32_t src_lkey, uint64_t src_addr,
                    size_t length)
.EE
.SH DESCRIPTION
The MLX5DV work request APIs (mlx5dv_wr_*) is an extension for IBV work
request API (ibv_wr_*) with mlx5 specific features for send work
request.
This may be used together with or without ibv_wr_* calls.
.SH USAGE
To use these APIs a QP must be created using mlx5dv_create_qp() with
\f[I]send_ops_flags\f[R] of struct ibv_qp_init_attr_ex set.
.PP
If the QP does not support all the requested work request types then QP
creation will fail.
.PP
The mlx5dv_qp_ex is extracted from the IBV_QP by ibv_qp_to_qp_ex() and
mlx5dv_qp_ex_from_ibv_qp_ex().
This should be used to apply the mlx5 specific features on the posted
WR.
.PP
A work request creation requires to use the ibv_qp_ex as described in
the man for ibv_wr_post and mlx5dv_qp with its available builders and
setters.
.SS QP Specific builders
.TP
\f[I]RC\f[R] QPs
\f[I]mlx5dv_wr_mr_interleaved()\f[R]
.RS
registers an interleaved memory layout by using an indirect mkey and
some interleaved data.
The layout of the memory pointed by the mkey after its registration will
be the \f[I]data\f[R] representation for the \f[I]num_interleaved\f[R]
entries.
This single layout representation is repeated by \f[I]repeat_count\f[R].
The \f[I]data\f[R] as described by struct mlx5dv_mr_interleaved will
hold real data defined by \f[I]bytes_count\f[R] and then a padding of
\f[I]bytes_skip\f[R].
Post a successful registration, RDMA operations can use this
\f[I]mkey\f[R].
The hardware will scatter the data according to the pattern.
The \f[I]mkey\f[R] should be used in a zero\-based mode.
The \f[I]addr\f[R] field in its \f[I]ibv_sge\f[R] is an offset in the
total data.
To create this \f[I]mkey\f[R] mlx5dv_create_mkey() should be used.
Current implementation requires the IBV_SEND_INLINE option to be on in
\f[I]ibv_qp_ex\->wr_flags\f[R] field.
To be able to have more than 3 \f[I]num_interleaved\f[R] entries, the QP
should be created with a larger WQE size that may fit it.
This should be done using the \f[I]max_inline_data\f[R] attribute of
\f[I]struct ibv_qp_cap\f[R] upon its creation.
As one entry will be consumed for strided header, the \f[I]mkey\f[R]
should be created with one more entry than the required
\f[I]num_interleaved\f[R].
In case \f[I]ibv_qp_ex\->wr_flags\f[R] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_UMR.
Unregister the \f[I]mkey\f[R] to enable another pattern registration
should be done via ibv_post_send with IBV_WR_LOCAL_INV opcode.
.RE
\f[I]mlx5dv_wr_mr_list()\f[R]
.RS
registers a memory layout based on list of ibv_sge.
The layout of the memory pointed by the \f[I]mkey\f[R] after its
registration will be based on the list of \f[I]sge\f[R] counted by
\f[I]num_sges\f[R].
Post a successful registration RDMA operations can use this
\f[I]mkey\f[R], the hardware will scatter the data according to the
pattern.
The \f[I]mkey\f[R] should be used in a zero\-based mode, the
\f[I]addr\f[R] field in its \f[I]ibv_sge\f[R] is an offset in the total
data.
Current implementation requires the IBV_SEND_INLINE option to be on in
\f[I]ibv_qp_ex\->wr_flags\f[R] field.
To be able to have more than 4 \f[I]num_sge\f[R] entries, the QP should
be created with a larger WQE size that may fit it.
This should be done using the \f[I]max_inline_data\f[R] attribute of
\f[I]struct ibv_qp_cap\f[R] upon its creation.
In case \f[I]ibv_qp_ex\->wr_flags\f[R] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_UMR.
Unregister the \f[I]mkey\f[R] to enable other pattern registration
should be done via ibv_post_send with IBV_WR_LOCAL_INV opcode.
.RE
.TP
\f[I]RC\f[R] or \f[I]DCI\f[R] QPs
\f[I]mlx5dv_wr_memcpy()\f[R]
.RS
Builds a DMA memcpy work request to copy data of length \f[I]length\f[R]
from \f[I]src_addr\f[R] to \f[I]dest_addr\f[R].
The copy operation will be done using the DMA MMO functionality of the
device to copy data on PCI bus.
The MLX5DV_QP_EX_WITH_MEMCPY flag in
\f[I]mlx5dv_qp_init_attr.send_ops_flags\f[R] needs to be set during QP
creation.
If the device or QP doesn\(cqt support it then QP creation will fail.
The maximum memcpy length that is supported by the device is reported in
\f[I]mlx5dv_context\->max_wr_memcpy_length\f[R].
A zero value in \f[I]mlx5dv_context\->max_wr_memcpy_length\f[R] means
the device doesn\(cqt support memcpy operations.
IBV_SEND_FENCE indicator should be used on a following send request
which is dependent on \f[I]dest_addr\f[R] of the memcpy operation.
In case \f[I]ibv_qp_ex\->wr_flags\f[R] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_MEMCPY.
.RE
.SS Raw WQE builders
.TP
\f[I]mlx5dv_wr_raw_wqe()\f[R]
It is used to build a custom work request (WQE) and post it on a normal
QP.
The caller needs to set all details of the WQE (except the
\(lqctrl.wqe_index\(rq and \(lqctrl.signature\(rq fields, which is the
driver\(cqs responsibility to set).
The MLX5DV_QP_EX_WITH_RAW_WQE flag in mlx5_qp_attr.send_ops_flags needs
to be set.
.RS
The wr_flags are ignored as it\(cqs the caller\(cqs responsibility to
set flags in WQE.
No matter what the send opcode is, the work completion opcode for a raw
WQE is IBV_WC_DRIVER2.
.RE
.SS QP Specific setters
.TP
\f[I]DCI\f[R] QPs
\f[I]mlx5dv_wr_set_dc_addr()\f[R] must be called to set the DCI WR
properties.
The destination address of the work is specified by \f[I]ah\f[R], the
remote DCT number is specified by \f[I]remote_dctn\f[R] and the DC key
is specified by \f[I]remote_dc_key\f[R].
This setter is available when the QP transport is DCI and send_ops_flags
in struct ibv_qp_init_attr_ex is set.
The available builders and setters for DCI QP are the same as RC QP.
DCI QP created with MLX5DV_QP_INIT_ATTR_MASK_DCI_STREAMS can call
\f[I]mlx5dv_wr_set_dc_addr_stream()\f[R] to define the
\f[I]stream_id\f[R] of the operation to allow HW to choose one of the
multiple concurrent DCI resources.
Calls to \f[I]mlx5dv_wr_set_dc_addr()\f[R] are equivalent to using
\f[I]stream_id\f[R]=0
.SH EXAMPLE
.IP
.EX
\f[I]/* create DC QP type and specify the required send opcodes */\f[R]
attr_ex.qp_type = IBV_QPT_DRIVER;
attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;

attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;
attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;

ibv_qp *qp = mlx5dv_create_qp(ctx, attr_ex, attr_dv);
ibv_qp_ex *qpx = ibv_qp_to_qp_ex(qp);
mlx5dv_qp_ex *mqpx = mlx5dv_qp_ex_from_ibv_qp_ex(qpx);

ibv_wr_start(qpx);

\f[I]/* Use ibv_qp_ex object to set WR generic attributes */\f[R]
qpx\->wr_id = my_wr_id_1;
qpx\->wr_flags = IBV_SEND_SIGNALED;
ibv_wr_rdma_write(qpx, rkey, remote_addr_1);
ibv_wr_set_sge(qpx, lkey, local_addr_1, length_1);

\f[I]/* Use mlx5 DC setter using mlx5dv_qp_ex object */\f[R]
mlx5dv_wr_set_wr_dc_addr(mqpx, ah, remote_dctn, remote_dc_key);

ret = ibv_wr_complete(qpx);
.EE
.SH SEE ALSO
\f[B]ibv_post_send\f[R](3), \f[B]ibv_create_qp_ex(3)\f[R],
\f[B]ibv_wr_post(3)\f[R], \f[B]mlx5dv_create_mkey(3)\f[R].
.SH AUTHOR
Guy Levi \c
.MT guyle@mellanox.com
.ME \c
.PP
Mark Zhang \c
.MT markzhang@nvidia.com
.ME \c
