commit fc13d676d07e5047177125cd3131c371eec8677b
Author: Nicolas Morey-Chaisemartin <nmoreychaisemartin@suse.com>
Date:   Tue Mar 9 11:24:04 2021 +0100

    Revert "rxe: Remove rxe_cfg"
    
    This reverts commit 0d2ff0e1502ebc63346bc9ffd37deb3c4fd0dbc9.

diff --git Documentation/rxe.md Documentation/rxe.md
index ea11fc4b2a27..8e753decb3cb 100644
--- Documentation/rxe.md
+++ Documentation/rxe.md
@@ -1,14 +1,22 @@
 # Configure Soft-RoCE (RXE):
 
+Load rdma_rxe kernel module using the rxe_cfg script included in the librxe RPM:
+
+	# rxe_cfg start (this might require sudo or root privileges)
+
 Create RXE device over network interface (e.g. eth0):
 
-	# rdma link add rxe_eth0 type rxe netdev eth0
+	# rxe_cfg add eth0
 
 Use the status command to display the current configuration:
+rxe_cfg status
+
+If configured successfully, you should see output similar to the following:
 
-	# rdma link
+```
+    Name  Link  Driver   Speed  NMTU  IPv4_addr  RDEV  RMTU
+    eth0  yes   mlx4_en                          rxe0  1024  (3)
+```
 
-If you are using a Mellanox HCA, make sure that the mlx4_ib/mlx5_ib kernel
-module is not loaded (modprobe –rv mlx4_ib) in the soft-RoCE machine.  Now you
-have an Infiniband device called “rxe0_eth0” that can be used to run any RoCE
-app.
+If you are using a Mellanox HCA: Need to make sure that the mlx4_ib kernel module is not loaded (modprobe –rv mlx4_ib) in the soft-RoCE machine.
+Now you have an Infiniband device called “rxe0” that can be used to run any RoCE app.
diff --git README.md README.md
index b31d448264e5..01d0f6a1c014 100644
--- README.md
+++ README.md
@@ -116,6 +116,8 @@ command above to work.
 You can use either `ibv_devices` or `rdma link` to verify that the device was
 successfully added.
 
+Use of `rxe_cfg` is deprecated, please use the `rdma` command instead.
+
 # Reporting bugs
 
 Bugs should be reported to the <linux-rdma@vger.kernel.org> mailing list
diff --git debian/rdma-core.install debian/rdma-core.install
index 564d4a1b6353..c60a76cfe416 100644
--- debian/rdma-core.install
+++ debian/rdma-core.install
@@ -21,6 +21,7 @@ lib/udev/rules.d/90-iwpmd.rules
 lib/udev/rules.d/90-rdma-hw-modules.rules
 lib/udev/rules.d/90-rdma-ulp-modules.rules
 lib/udev/rules.d/90-rdma-umad.rules
+usr/bin/rxe_cfg
 usr/lib/truescale-serdes.cmds
 usr/sbin/iwpmd
 usr/sbin/rdma-ndd
@@ -33,3 +34,4 @@ usr/share/man/man5/iwpmd.conf.5
 usr/share/man/man7/rxe.7
 usr/share/man/man8/iwpmd.8
 usr/share/man/man8/rdma-ndd.8
+usr/share/man/man8/rxe_cfg.8
diff --git providers/rxe/CMakeLists.txt providers/rxe/CMakeLists.txt
index d8f3265176e4..15dc627e4e67 100644
--- providers/rxe/CMakeLists.txt
+++ providers/rxe/CMakeLists.txt
@@ -1,3 +1,8 @@
 rdma_provider(rxe
   rxe.c
   )
+rdma_subst_install(FILES "rxe_cfg.in"
+  RENAME "rxe_cfg"
+  DESTINATION "${CMAKE_INSTALL_BINDIR}"
+  PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE
+  )
diff --git providers/rxe/man/CMakeLists.txt providers/rxe/man/CMakeLists.txt
index 53d78dbe7de1..69e8bd8cf20c 100644
--- providers/rxe/man/CMakeLists.txt
+++ providers/rxe/man/CMakeLists.txt
@@ -1,3 +1,4 @@
 rdma_man_pages(
   rxe.7
+  rxe_cfg.8
 )
diff --git providers/rxe/man/rxe.7 providers/rxe/man/rxe.7
index 474ffff120a7..79349e9d3b50 100644
--- providers/rxe/man/rxe.7
+++ providers/rxe/man/rxe.7
@@ -6,7 +6,7 @@ rxe \- Software RDMA over Ethernet
 .SH "SYNOPSIS"
 \fBmodprobe rdma_rxe\fR
 .br
-This is usually performed by a configuration utility (see \fBrdma link\fR(8).)
+This is usually performed by a configuration utility (see \fBrxe_cfg\fR(8).)
 
 .SH "DESCRIPTION"
 The rdma_rxe kernel module provides a software implementation of the RoCEv2
@@ -23,9 +23,17 @@ In particular, while the use of a GRH header is optional within IB subnets, it i
 \fB/sys/class/infiniband/rxe[0,1,...]\fR
 Directory that holds RDMA device information. The format is the same as other RDMA devices.
 
+.TP
+\fB/sys/module/rdma_rxe_net/parameters/add\fR
+Write only file used by \fBrxe_cfg(8)\fR to add new RXE devices to existing Ethernet devices.
+
+.TP
+\fB/sys/module/rdma_rxe_net/parameters/remove\fR
+Write only file used by \fBrxe_cfg(8)\fR to remove RXE devices.
+
 .TP
 \fB/sys/module/rdma_rxe_net/parameters/mtu\fR
-Write only file used to configure RoCE and Ethernet MTU values.
+Write only file used by \fBrxe_cfg(8)\fR to configure RoCE and Ethernet MTU values.
 
 .TP
 \fB/sys/module/rdma_rxe/parameters/max_ucontext\fR
@@ -90,7 +98,7 @@ Read/Write file that controls the maximum gap between the PSN of request packets
 Read/Write file that controls the default mtu used for UD packets.
 
 .SH "SEE ALSO"
-.BR rdma (8),
+.BR rxe_cfg (8),
 .BR verbs (7),
 
 .SH "AUTHORS"
diff --git providers/rxe/man/rxe_cfg.8 providers/rxe/man/rxe_cfg.8
new file mode 100644
index 000000000000..c29c3d009306
--- /dev/null
+++ providers/rxe/man/rxe_cfg.8
@@ -0,0 +1,104 @@
+.\" -*- nroff -*-
+.\"
+.TH RXE_CFG 8 2011-06-29 1.0.0
+.SH "NAME"
+rxe_cfg \- rxe configuration tool for RXE (Soft RoCE)
+.SH "SYNOPSIS"
+\fBrxe_cfg [status]\fR
+.br
+\fBrxe_cfg start\fR [\fB\-p\fR \fIproto\fR]
+.br
+\fBrxe_cfg stop\fR
+.br
+\fBrxe_cfg persistent\fR
+.br
+\fBrxe_cfg add\fR [\fB\-n\fR] \fIethN\fR
+.br
+\fBrxe_cfg remove\fR [\fB\-n\fR] \fIethN\fR|\fIrxeN\fR
+.br
+\fBrxe_cfg crc enable\fR|\fBdisable\fR
+.br
+\fBrxe_cfg\fR \fBmtu\fR [\fB\-f\fR] [\fIrxeN\fR] \fImtu_size\fR
+.br
+.SH "DESCRIPTION"
+
+.I
+Note
+This command is deprecated. Please use
+.B
+rdma-link(8)
+instead.
+
+rxe_cfg is the configuration tool for the RXE software implementation of the RoCE protocol.
+
+The RXE kernel modules are loaded, configured, reconfigured and unloaded via the various rxe_cfg command options, documented below.
+
+.SH "PARAMETERS"
+.TP
+\fIproto\fR
+Ethertype field. Default value is 0x8915. This value must be changed to use RXE on Mellanox ConnectX adapters.
+
+.TP
+\fIethN\fR
+Network device name as listed in /sys/class/net. Only Ethernet devices are supported; ie. eth0 or eth0.1234 for VLANs.
+
+.TP
+\fIrxeN\fR
+RXE device name as listed in /sys/class/infiniband/. Examples are rxe0 or rxe1.
+
+.TP
+\fImtu_size\fR
+RoCE mtu. For RoCE the mtu represents the payload excluding headers and has the possible values: 256, 512, 1024, 2048 and 4096.
+
+.SH "COMMANDS"
+.TP
+[\fBstatus\fR]
+The \fBstatus\fR command prints a table of information on available Ethernet devices and configured RXE instances.  The status display is the default if no options are provided.
+
+.TP
+\fBstart\fR [\fB\-p\fR \fIproto\fR]
+The \fBstart\fR command loads the RXE modules and configures any persistent instances.  If the \fB-p\fR \fIproto\fR option is included, the RXE modules will be configured to use Ethertype = \fIproto\fR.  (This allows testing RXE on devices, like Mellanox ConnectX, that already support the default RoCE Ethertype in hardware.)
+
+.TP
+\fBstop\fR
+The \fBstop\fR command unconfigures all RXE instances and attempts to unload the kernel modules.
+
+.TP
+\fBpersistent\fR
+The \fBpersistent\fR command prints the list of Ethernet devices for which a RXE instance is persistently configured.
+
+.TP
+\fBadd\fR [\fB\-n\fR] \fIethN\fR
+The \fBadd\fR command will configure a RXE instance on Ethernet device \fIethN\fR (e.g. eth0).  The RXE modules must have already been loaded via \fBrxe_cfg start\fR.
+
+The default behavior is to add \fIethN\fR to a file of persistent configurations and the same RXE device will be configured the next time that \fBrxe_cfg start\fR is run.  If the \fB-n\fR option is included the device is not added to the persistence file.
+
+.TP
+\fBremove\fR [\fB\-n\fR] \fIethN\fR|\fIrxeN\fR
+The \fBremove\fR command will remove the specified RXE instance.  The parameter must match a currently active ethN or rxeN name.
+
+If the \fB-n\fR option is included the RXE device will be removed but not removed from the persistent state. So it will be recreated the next time that \fBrxe_cfg start\fR is run.
+
+.TP
+\fBmtu\fR [\fB\-f\fR] [\fIrxeN\fR] \fImtu_size\fR
+The \fBmtu\fR command will set the RoCE MTU of all RXE devices to \fImtu_size\fR, provided that the underlying Ethernet MTU is sufficiently large.  If the Ethernet MTU is not large enough, RXE will use the largest MTU that fits; the driver remembers the requested RoCE MTU and will increase the currently active MTU if the Ethernet MTU is later changed up to the requested MTU.
+
+If the \fB-f\fR option is included the underlying Ethernet MTUs will be increased if necessary to the minimum size to accommodate a RoCE MTU of \fIsize\fR.
+
+If a \fIrxeN\fR instance is specified then only that instance will be affected by the command.
+
+.TP
+\fBcrc\fR \fBenable\fR|\fBdisable\fR
+The \fBcrc\fR command will enable or disable RoCE ICRC calculation.  Valid ICRCs are be required to communicate hardware RoCE NICs, but when a RXE instance is communicating with another RXE instance the performance will be better with the CRC disabled.
+
+This option is global to all RXE instances.
+
+.SH "FILES"
+.TP
+\fB[PREFIX]/etc/rxe.conf\fR
+RXE configuration file. Contains the list of persistent RXE instances.  All persistent RXE instances can be removed by deleting this file (note this will take effect on the next "rxe_cfg start" -- to remove actively configured instances, you must "rxe_cfg stop").
+
+.SH "SEE ALSO"
+.BR rxe (7),
+.SH "AUTHORS"
+Written by John Groves, Frank Zago and Bob Pearson at System Fabric Works.
diff --git providers/rxe/rxe_cfg.in providers/rxe/rxe_cfg.in
new file mode 100755
index 000000000000..96126d3ae653
--- /dev/null
+++ providers/rxe/rxe_cfg.in
@@ -0,0 +1,677 @@
+#!/usr/bin/perl
+
+# * Copyright (c) 2009-2011 Mellanox Technologies Ltd. All rights reserved.
+# * Copyright (c) 2009-2011 System Fabric Works, Inc. All rights reserved.
+# *
+# * This software is available to you under a choice of one of two
+# * licenses.  You may choose to be licensed under the terms of the GNU
+# * General Public License (GPL) Version 2, available from the file
+# * COPYING in the main directory of this source tree, or the
+# * OpenIB.org BSD license below:
+# *
+# *     Redistribution and use in source and binary forms, with or
+# *     without modification, are permitted provided that the following
+# *     conditions are met:
+# *
+# *	- Redistributions of source code must retain the above
+# *	  copyright notice, this list of conditions and the following
+# *	  disclaimer.
+# *
+# *	- Redistributions in binary form must reproduce the above
+# *	  copyright notice, this list of conditions and the following
+# *	  disclaimer in the documentation and/or other materials
+# *	  provided with the distribution.
+# *
+# * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# * SOFTWARE.
+#
+
+use warnings;
+use strict;
+
+use File::Basename;
+use File::Path qw(make_path);
+use Getopt::Long;
+
+my $help = 0;
+my $no_persist = 0;
+my $debug = 0;
+my $force = 0;
+my $linkonly = 0;
+my $parms = "/sys/module/rdma_rxe/parameters";
+my $modprobe_opt = "";
+my $modprobe_checked = "0";
+my $persistence_path = "@CMAKE_INSTALL_FULL_SHAREDSTATEDIR@/rxe";
+my $persistence_file = "${persistence_path}/rxe";
+my $num_persistent = 0;
+my $sys = "/sys/module/rdma_rxe/parameters";
+my %rxe_names;
+my @rxe_array;
+my %eth_names;
+my @eth_list;
+my %eth_driver;
+my %link_state;
+my %link_speed;
+my %eth_mtu;
+my %ipv4_addr;
+my %rxe_mtu;
+my @persistence_array;
+my %persistence_hash;
+my @mlx4_port;
+my @mlx4_ether;
+my @roce_list;
+
+# Read a file and return its contents as a string.
+sub read_file {
+    my $filename = shift;
+    my $result = "";
+
+    if (open(FILE, $filename)) {
+	$result = <FILE>;
+	close FILE;
+    }
+    return $result;
+}
+
+#get mapping between rxe and eth devices
+sub get_names {
+    my $i = 0;
+    
+    foreach my $rxe (glob("/sys/class/infiniband/rxe*")) {
+	$rxe = basename($rxe);
+	my $eth = read_file("/sys/class/infiniband/$rxe/parent");
+	chomp($eth);
+	
+	if (($eth =~ /[\w]+[\d]/)
+	    && ($rxe =~ /rxe[0123456789]/)) {
+	    
+	    # hash ethername to rxename
+	    $rxe_names{$eth} = $rxe;
+	    $rxe_array[$i++] = $rxe;
+	    
+	    # hash rxename to ethername
+	    $eth_names{$rxe} = $eth;
+	}
+    }
+}
+
+# get list of Mellanox RoCE ports
+sub get_mlx4_list {
+    my $i = 0;
+
+    foreach my $mlx4 (glob("/sys/class/infiniband/mlx4_*")) {
+	$mlx4 = basename($mlx4);
+	foreach my $port (glob("/sys/class/infiniband/$mlx4/ports/*")) {
+	    $port = basename($port);
+	    my $link = read_file("$port/link_layer");
+	    chomp($link);
+
+	    if ($link =~ "Ethernet") {
+		$roce_list[$i++] = "$mlx4:$port";
+	    }
+	}
+    }
+}
+
+#collect per device information
+sub get_dev_info {
+    my @list;
+    my @fields;
+    my @lines;
+    my $line;
+    my $eth;
+    my $drv;
+    my $np;
+    my $i = 0;
+    my $j = 0;
+
+    get_mlx4_list();
+
+    my @my_eth_list = ();
+    foreach my $my_eth_dev (glob("/sys/class/net/*")) {
+       $my_eth_dev = basename($my_eth_dev);
+          if ($my_eth_dev ne "bonding_masters"){
+             my $my_dev_type = read_file("/sys/class/net/${my_eth_dev}/type");
+             chomp($my_dev_type);
+             if ($my_dev_type == "1") {
+                push(@my_eth_list, "$my_eth_dev");
+             }
+          }
+    }
+
+    @list = @my_eth_list;
+    foreach $eth (@list) {
+	chomp($eth);
+
+	$eth_list[$i++] = $eth;
+
+	@lines = `ethtool -i $eth`;
+	foreach $line (@lines) {
+	    chomp($line);
+
+	    @fields = split(/\s+/, $line);
+	    chomp($fields[0]);
+
+	    if ($fields[0] =~ /driver:/) {
+		$drv = $fields[1];
+		$eth_driver{$eth} = $drv;
+
+		if ($drv =~ /mlx4_en/ && scalar(@roce_list) > 0 ) {
+		    $eth_names{$roce_list[$j++]} = $eth;
+		}
+	    }
+	}
+
+	# get link status
+	$link_state{$eth} = "";
+	$link_speed{$eth} = "";
+
+	@lines = `ethtool $eth`;
+	foreach $line (@lines) {
+	    chomp($line);
+
+	    @fields = split(/:/, $line);
+	    if (defined($fields[1])) {
+		    $fields[1] =~ s/^\s+//g;
+		    if ($fields[0] =~ "Link detected") {
+			$link_state{$eth} = $fields[1];
+		    }
+	    }
+	    elsif ($line =~ "10000baseT") {
+		$link_speed{$eth} = "10GigE";
+	    }
+	}
+
+	$ipv4_addr{$eth} = "            ";
+	$eth_mtu{$eth} = "";
+
+	@lines = `ip addr show $eth`;
+	foreach $line (@lines) {
+		# get IP address
+		if ($line =~ /inet /) {
+			$line =~ s/^\s+inet ([0-9.]+)\//$1 /g;
+			@fields = split(/\s+/, $line);
+			$ipv4_addr{$eth} = $fields[0];
+		}
+
+		# get ethernet mtu
+		if ($line =~ /mtu /) {
+			$line =~ s/^.*mtu //g;
+			@fields = split(/\s+/, $line);
+			$eth_mtu{$eth} = $fields[0];
+		}
+    }
+    }
+
+    # get rxe mtu
+    foreach my $rxe (@rxe_array) {
+	
+	@lines = `ibv_devinfo -d $rxe`;
+	foreach $line (@lines) {
+	    if ($line =~ "active_mtu") {
+		$line =~ s/^\s+active_mtu:\s+//g;
+		chomp($line);
+
+		$rxe_mtu{$rxe} = $line;
+	    }
+	}
+	$rxe_mtu{$rxe} = "(?)" if (!$rxe_mtu{$rxe});
+    }
+}
+
+# return string or the string "###" if string is all whitespace
+sub set_field {
+    my $fld = $_[0];
+
+    if (defined($fld) && $fld =~ /\S/) {
+        return $fld;
+    } else {
+        return "###";
+    }
+}
+
+# format status output into fixed width columns
+sub status_print {
+    my @fields;
+    my $field;
+    my @flen = ();
+    my $num_fields = 0;
+    my $i;
+    my $pad;
+    my $line;
+
+    # one pass to size the columns
+    foreach $line (@_) {
+	@fields = split(/\s+/, $line);
+	$i = 0;
+	foreach $field (@fields) {
+	    if (!defined($flen[$i])) {
+		$flen[$i] = length($field);
+	    }
+	    else {
+		$flen[$i] = max($flen[$i], length($field));
+	    }
+	    $i++;
+	}
+
+	if ($i > $num_fields) {
+	    $num_fields = $i;
+	}
+    }
+
+    # one pass to print
+    foreach $line (@_) {
+	print "  ";
+	@fields = split(/\s+/, $line);
+	for ($i = 0; $i < $num_fields; $i++) {
+	    if (defined($fields[$i])) {
+	        $pad = $flen[$i] - length($fields[$i]) + 2;
+	    }
+	    else {
+	        $pad = $flen[$i] + 2;
+	    }
+	    if (defined($fields[$i]) && ($fields[$i] ne "###")) {
+		print "$fields[$i]";
+	    }
+	    else {
+		print "   ";
+	    }
+	    printf("%*s", $pad, "");
+	}
+	print "\n";
+    }
+}
+
+# check driver load status
+sub check_module_status {
+    if (-e $sys) {
+	return 0;
+    } else {
+	return 1;
+    }
+}
+
+# print driver load status and ethertype for rdma_rxe and rdma_rxe_net
+sub show_module_status {
+    print "rdma_rxe module not loaded\n" if (!(-e $sys));
+}
+
+# print rxe status
+sub do_status {
+    my $instance = $_[0];
+    my $ln = 0;
+    my @outp;
+    my $rxe;
+    my $rmtu;
+
+    get_names();
+    get_dev_info();
+    show_module_status();
+
+    $outp[$ln++] = "Name\tLink\tDriver\t\tSpeed\tNMTU\tIPv4_addr\tRDEV\tRMTU";
+
+    foreach my $eth (@eth_list) {
+
+	# handle case where rxe_drivers are not loaded
+	if (defined($rxe_names{$eth})) {
+		$rxe = $rxe_names{$eth};
+		$rmtu = $rxe_mtu{$rxe};
+	}
+	else {
+		$rxe = "";
+		$rmtu = "";
+	}
+
+	if ((!defined($instance) 
+	     && (($linkonly == 0) || ($link_state{$eth} =~ "yes")))
+	    || (defined($instance) && ($rxe =~ "$instance"))) {
+	    $outp[$ln] =  set_field("$eth");
+	    $outp[$ln] .= "\t";
+	    $outp[$ln] .= set_field("$link_state{$eth}");
+	    $outp[$ln] .= "\t";
+	    $outp[$ln] .= set_field(exists($eth_driver{$eth}) ? $eth_driver{$eth} : "");
+	    $outp[$ln] .= "\t";
+	    $outp[$ln] .= set_field("$link_speed{$eth}");
+	    $outp[$ln] .= "\t";
+	    $outp[$ln] .= set_field("$eth_mtu{$eth}");
+	    $outp[$ln] .= "\t";
+	    $outp[$ln] .= set_field("$ipv4_addr{$eth}");
+	    $outp[$ln] .= "\t";
+	    $outp[$ln] .= set_field("$rxe");
+	    $outp[$ln] .= "\t";
+	    $outp[$ln] .= set_field("$rmtu");
+	    $ln++;
+	}
+    }
+
+    status_print(@outp);
+}
+
+# read file containing list of ethernet devices into a list
+sub populate_persistence {
+    my $i = 0;
+    
+    open FILE, $persistence_file;
+    while(<FILE>) {
+	my $line = $_;
+	chomp($line);
+	$line =~ s/^\s+//g;
+	if ($line =~ /[\w]+[\d]/) {
+	    # in case we add fields later
+	    my ($eth, $cruft) = split(/\s+/, $line, 2);
+	    if ($eth =~ /^[\w]+[\d]/) {
+		$persistence_array[$i] = $eth;
+		$persistence_hash{$eth} = $i++;
+	    }
+	}
+    }
+    close FILE;
+
+    $num_persistent = $i;
+}
+
+# print out list of ethernet devices to file
+sub commit_persistent {
+    my $i;
+    my $eth;
+
+    open(PF, ">$persistence_file");
+    
+    for ($i = 0; $i < $num_persistent; $i++) {
+	$eth = $persistence_array[$i];
+	if ($eth =~ /[\w]+[\d]/) {
+	    print(PF "$persistence_array[$i]\n");
+	}
+    }
+
+    close(PF);
+}
+
+sub delete_persistent {
+    my $eth = $_[0];
+    
+    if (defined($persistence_hash{$eth})) {
+	$persistence_array[$persistence_hash{$eth}] = "";
+    }
+}
+
+sub add_persistent {
+    my $eth = $_[0];
+
+    # Is this one already in the persistence list?
+    if (!defined($persistence_hash{$eth})) {
+	$persistence_array[$num_persistent] = $eth;
+	$persistence_hash{$eth} = $num_persistent;
+	$num_persistent++;
+    }
+}
+
+# add new rxe device to eth if not already up
+sub rxe_add {
+    my $eth = $_[0];
+
+    if (!($eth =~ /[\w]+[\d]/)) {
+	print "eth_name ($eth) looks bogus\n";
+	return;
+    }
+
+    if (!defined($rxe_names{$eth})) {
+	system("echo '$eth' > $parms/add");
+    }
+    if (!$no_persist) {
+	add_persistent($eth);
+	commit_persistent();
+    }
+}
+
+sub rxe_remove {
+    my $arg2 = $_[0];
+    my $rxe;
+    my $eth;
+
+    print "remove $arg2\n"  if ($debug > 0);
+
+    if ($arg2 =~ /[\w]+[\d]/) {
+	$eth = $arg2;
+	$rxe = $rxe_names{$eth};
+    }
+    elsif ($arg2 =~ /rxe[0123456789]/) {
+	$rxe = $arg2;
+	$eth = $eth_names{$rxe};
+    }
+    elsif ($arg2 eq "all") {
+	$rxe = "all";
+    }
+
+    if (($rxe eq "all") || ($rxe =~ /^rxe[0123456789]/)) {
+	my $cmd = "echo '$rxe' > $parms/remove";
+	#print "$cmd\n";
+	system($cmd);
+	if (!$no_persist) {
+	    if ($rxe eq "all") {
+		unlink($persistence_file);
+	    }
+	    elsif ($eth =~/[\w]+[\d]/) {
+		delete_persistent($eth);
+		commit_persistent();
+	    }
+	    else {
+		print "Warning: Unable to resolve ethname; "
+		    . "instance may persist on restart\n";
+	    }
+	}
+    }
+    else {
+	print "rxe instance $rxe not found\n";
+    }
+}
+
+sub get_devinfo {
+    my $rxe = $_[0];
+
+    my $cmd = "ibv_devinfo -d $rxe";
+    return `$cmd`;
+}
+
+# allow unsupported modules to load in SLES11 if allowed
+sub modprobe {
+    my $module = $_[0];
+    my $opts = $_[1];
+    my @lines;
+    my $line;
+
+    if ($modprobe_checked == "0") {
+	@lines = `modprobe -c`;
+	foreach $line (@lines) {
+	    if ($line =~ /^allow_unsupported_modules  *0/) {
+		$modprobe_opt = " --allow-unsupported-modules ";
+		last;
+	    }
+	}
+	$modprobe_checked = "1";
+    }
+
+    if (!defined($opts)) {
+	$opts = "";
+    }
+
+    system("modprobe $modprobe_opt $module $opts");
+}
+
+# bring up rxe
+sub do_start {
+    my $proto_str = "";
+
+    system("mkdir -p $persistence_path");
+    system("touch $persistence_file");
+
+    modprobe("ib_core");
+    modprobe("ib_uverbs");
+    modprobe("rdma_ucm");
+    modprobe("rdma_rxe");
+
+    populate_persistence();
+    system("udevadm control --reload");
+
+    foreach my $eth (@persistence_array) {
+	rxe_add($eth);
+    }
+
+    get_names();
+
+    foreach my $rxe (@rxe_array) {
+	my $stat = get_devinfo($rxe);
+	if ($stat =~ "PORT_DOWN") {
+		my $cmd = "ip link set $eth_names{$rxe} up";
+		system($cmd);
+	}
+    }
+
+}
+
+# check if argument is an integer
+sub is_integer {
+    defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
+}
+
+# remove all rxe devices and unload drivers
+sub do_stop {
+    my $rxe;
+
+    foreach $rxe (@rxe_array) {
+	system("echo '$rxe' > $sys/remove");
+    }
+
+    if (-e $sys) {
+	system("rmmod rdma_rxe");
+    }
+
+    if (-e $sys) {
+	print "unable to unload drivers, reboot required\n";
+    }
+}
+
+sub do_debug {
+    my $arg2 = $_[0];
+    my $debugfile = "$parms/debug";
+    chomp($arg2);
+
+    if (!(-e "$debugfile")) {
+	print "Error: debug is compiled out of this rxe driver\n";
+	return;
+    }
+
+    if    ($arg2 eq "on")  { system("echo '31' > $debugfile"); }
+    elsif ($arg2 eq "off") { system("echo '0'  > $debugfile"); }
+    elsif ($arg2 eq "0")   { system("echo '0'  > $debugfile"); }
+    elsif ($arg2 eq "")    { }
+	elsif ($arg2 ge "0" && $arg2 le "31") {
+	    system("echo '$arg2' > $debugfile");
+	}
+	else {
+	    print "unrecognized debug cmd ($arg2)\n";
+	}
+
+    my $current = read_file($debugfile);
+    chomp($current);
+    if ($current > 0) {
+	print "Debug is ON ($current)\n";
+    }
+    elsif ($current == 0) {
+	print "Debug is OFF\n";
+    }
+    else {
+	print "Unrecognized debug value\n";
+    }
+}
+
+sub max {
+    my $a = $_[0];
+    my $b = $_[1];
+    return $a if ($a > $b);
+    return $b;
+}
+
+# show usage for rxe_cfg
+sub usage {
+    print "  Usage:\n";
+    print "    rxe_cfg [options] start|stop|status|persistent\n";
+    print "    rxe_cfg debug on|off|<num>\n";
+    print "    rxe_cfg [-n] add <ndev>\n";
+    print "    rxe_cfg [-n] remove <ndev>|<rdev>\n";
+    print "\n";
+    print "    <ndev> = network device e.g. eth3\n";
+    print "    <rdev> = rdma device e.g. rxe1\n";
+    print "\n";
+    print "  Options:\n";
+    print "    -h: print this usage information\n";
+    print "    -n: do not make the configuration action persistent\n";
+    print "    -v: print additional debug output\n";
+    print "    -l: show status for interfaces with link up\n";
+    print "    -p <num>: (start command only) - set ethertype\n";
+}
+
+sub main {
+    GetOptions(
+	   "-h"          => \$help,
+	   "--help"      => \$help,
+	   "-n"          => \$no_persist,
+	   "-v:+"        => \$debug,
+	   "-f"          => \$force,
+	   "-l"          => \$linkonly,
+	   );
+
+    my $arg1 = $ARGV[0];
+    my $arg2 = $ARGV[1];
+    my $arg3 = $ARGV[2];
+
+    # status is the default
+    if (!defined($arg1) || ($arg1 =~ /status/)) {
+        do_status($arg2);
+        exit;
+    }
+
+    if ($help) {
+        usage();
+        exit;
+    }
+
+    # stuff that does not require modules to be loaded
+    if    ($arg1 eq "help")       { usage(); exit; }
+    elsif ($arg1 eq "start")      { do_start(); do_status(); exit; }
+    elsif ($arg1 eq "persistent") { system("cat $persistence_file"); exit; }
+
+
+    # can't do much else, bail if modules aren't loaded
+    if (check_module_status()) {
+	exit;
+    }
+
+    # create persistence file if necessary
+    make_path($persistence_path);
+    if (!(-e $persistence_file)) {
+        `touch $persistence_file`;
+    }
+
+    # Get full context of the configuration
+    populate_persistence();
+    get_names();
+    get_dev_info();
+
+    # Stuff that requires the rdma_rxe module to be loaded
+    if    ($arg1 eq "stop")   { do_stop(); 	   exit; }
+    elsif ($arg1 eq "debug")  { do_debug($arg2);   exit; }
+    elsif ($arg1 eq "add")    { rxe_add($arg2);    exit; }
+    elsif ($arg1 eq "remove") { rxe_remove($arg2); exit; }
+    elsif ($arg1 eq "help")   { usage();	   exit; }
+}
+
+main();
+
+exit;
diff --git redhat/rdma-core.spec redhat/rdma-core.spec
index 0d4da2ba9503..466433134317 100644
--- redhat/rdma-core.spec
+++ redhat/rdma-core.spec
@@ -428,10 +428,12 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh
 %{_libexecdir}/rdma-set-sriov-vf
 %{_libexecdir}/mlx4-setup.sh
 %{_libexecdir}/truescale-serdes.cmds
+%{_bindir}/rxe_cfg
 %{_sbindir}/rdma-ndd
 %{_unitdir}/rdma-ndd.service
 %{_mandir}/man7/rxe*
 %{_mandir}/man8/rdma-ndd.*
+%{_mandir}/man8/rxe*
 %license COPYING.*
 
 %files devel
diff --git suse/rdma-core.spec suse/rdma-core.spec
index e49b07abc34a..566aed0497ef 100644
--- suse/rdma-core.spec
+++ suse/rdma-core.spec
@@ -198,7 +198,8 @@ Requires:       %{efa_lname} = %{version}-%{release}
 Requires:       %{mlx4_lname} = %{version}-%{release}
 Requires:       %{mlx5_lname} = %{version}-%{release}
 %endif
-# Recommended packages for rxe
+# Recommended packages for rxe_cfg
+Recommends:     ethtool
 Recommends:     iproute2
 
 %description -n libibverbs
@@ -654,7 +655,9 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh
 %doc %{_docdir}/%{name}-%{version}/libibverbs.md
 %doc %{_docdir}/%{name}-%{version}/rxe.md
 %doc %{_docdir}/%{name}-%{version}/tag_matching.md
+%{_bindir}/rxe_cfg
 %{_mandir}/man7/rxe*
+%{_mandir}/man8/rxe*
 
 %files -n libibnetdisc%{ibnetdisc_major}
 %defattr(-, root, root)
