Mellanox ConnectX-3 Pro UEFI iPXE boot

  Device Type:      ConnectX3Pro
  Part Number:      MCX312B-XCC_Ax
  Description:      ConnectX-3 Pro EN network interface card; 10GigE; dual-port SFP+; PCIe3.0 x8 8GT/s; RoHS R6
  PSID:             MT_1200111023
  PCI Device Name:  /dev/mst/mt4103_pci_cr0
  Port1 MAC:        ec0d9a00aab1
  Port2 MAC:        ec0d9a00aab2
  Versions:         Current        Available    
     FW             2.42.5000      N/A          
     PXE            3.4.0752       N/A          

Flash UEFI firmware
Request UEFI firmware from support@mellanox.com
http://www.panticz.de/mellanox/firmware-update

flint -y -d /dev/mst/mt4103_pci_cr0 -i firmware fw-ConnectX3Pro-rel-2_42_5000-MCX312B-XCC_Ax-FlexBoot-3.4.752-UEFI-14.11.46.bin b
  Device Type:      ConnectX3Pro
  Part Number:      MCX312B-XCC_Ax
  Description:      ConnectX-3 Pro EN network interface card; 10GigE; dual-port SFP+; PCIe3.0 x8 8GT/s; RoHS R6
  PSID:             MT_1200111023
  PCI Device Name:  /dev/mst/mt4103_pci_cr0
  Port1 MAC:        ec0d9a00aab1
  Port2 MAC:        ec0d9a00aab2
  Versions:         Current        Available    
     FW             2.42.5000      N/A          
     PXE            3.4.0752       N/A          
     UEFI           14.11.0046     N/A     

Fix iPXE boot issue (recompile ipxe.efi) when connected to LACP swith port
http://www.panticz.de/ipxe/compile

Move Elasticsearch data to dedicated LV

# create lvm
pvcreate /dev/disk/by-id/ata-INTEL_SSDSC2KB076T8_*
vgcreate data /dev/disk/by-id/ata-INTEL_SSDSC2KB076T8_*
lvcreate --name elasticsearch --size 2T data
mkfs.ext4 /dev/data/elasticsearch
 
# pre-sync data
mount /dev/data/elasticsearch /mnt/
rsync -aHAXx --numeric-ids /var/lib/docker/volumes/elasticsearch/ /mnt/
 
# sync data
docker stop elasticsearch
rsync --delete -aHAXxv --numeric-ids /var/lib/docker/volumes/elasticsearch/ /mnt/
rsync --delete -aHAXxv --numeric-ids /var/lib/docker/volumes/elasticsearch/ /mnt/
umount /mnt
 
# mount new LV

SSH reverse tunel over public host

@Office

cat <<EOF> ~/bin/proxy-ssh-forward.sh
#!/bin/bash
 
eval \$(ssh-agent) && ssh-add
 
while true; do
    echo "Connect to proxy ..."
    ssh -a -v -N -R 7422:localhost:22 -o ServerAliveInterval=30 -o ServerAliveCountMax=10 proxy.example.com
    sleep 10
done
EOF
 
chmod +x ~/bin/proxy-ssh-forward.sh
nohup ~/bin/proxy-ssh-forward.sh

@Home

cat <<EOF>> ~/.ssh/config.d/office
Host proxy.example.com
Hostname 1.2.3.4
User foo
 
Host proxy-ssh-office
Hostname 127.0.0.1
Port 7422
User bar
#ForwardAgent yes
ProxyCommand ssh proxy.example.com -W %h:%p
EOF
 
ssh proxy-ssh-office

router / dnsmasq

CONTAINER=router
 
lxc launch ubuntu:18.04 ${CONTAINER}
lxc file push --uid 0 --gid 0 --mode 600 ~/.ssh/id_rsa.pub ${CONTAINER}/root/.ssh/authorized_keys
lxc exec ${CONTAINER} -- bash -c "apt update -qq && apt dist-upgrade -y"
 
apt install dnsmasq
 
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
sysctl -w net.ipv4.ip_forward=1
 
root@router:~# cat  /etc/netplan/50-cloud-init.yaml 
network:
    version: 2
    ethernets:
        eth0:
            dhcp4: true
        eth1:
            dhcp4: no
            addresses:
              - 192.168.0.1/24
 
 
 
# cat /etc/systemd/system/masquerade.service
[Unit]
Description=Enable MASQUERADE
After=network.target
 
[Service]
ExecStart=/sbin/iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
 
[Install]
WantedBy=multi-user.target
 
 
systemctl enable masquerade.service
 
 
root@router:~# diff /etc/dnsmasq.conf.org /etc/dnsmasq.conf
106c106

Deploy OpenStack host with Ironic and Redfish

Define node variables

NODE=com4-dev
NODE_BMC_HOST=com4-dev.ipmi.dev.i.example.com
NODE_MAC_NIC1=00:11:22:33:44:55

Define env variables

NODE_BMC_USER=ADMIN
NODE_BMC_PASS=ADMIN

Create now node with Redfish (pxe device boot broken)
https://docs.openstack.org/ironic/latest/admin/drivers/redfish.html

openstack baremetal node create \
  --name ${NODE} \
  --driver redfish \
  --driver-info redfish_address="https://${NODE_BMC_HOST}" \
  --driver-info redfish_username=${NODE_BMC_USER} \
  --driver-info redfish_password=${NODE_BMC_PASS} \
  --driver-info redfish_verify_ca=false \
  --driver-info redfish_system_id=/redfish/v1/Systems/1

Create now node with IPMI
https://docs.openstack.org/ironic/latest/admin/drivers/ipmitool.html

openstack baremetal node create \
  --name ${NODE} \
  --driver ipmi \
  --driver-info ipmi_address=${NODE_BMC_HOST} \
  --driver-info ipmi_username=${NODE_BMC_USER} \
  --driver-info ipmi_password=${NODE_BMC_PASS} 

iPXE
https://docs.openstack.org/ironic/latest/admin/interfaces/boot.html#pxe-boot