OpenStack: Regenerate kolla password file 

OS_ENV=dev

rm /etc/kolla/passwords.yml
cp /etc/kolla/config/passwords-${OS_ENV}.yml /etc/kolla/passwords.yml

# marge new password / reorder passwords
# kolla-genpwd -p /etc/kolla/passwords.yml
# cp /etc/kolla/passwords.yml /etc/kolla/config/passwords-${OS_ENV}.yml

cat /etc/kolla/config/passwords-${OS_ENV}.yml | cut -d" " -f1 > /etc/kolla/passwords.yml
kolla-genpwd -p /etc/kolla/passwords.yml
mv /etc/kolla/passwords.yml /etc/kolla/config/passwords-${OS_ENV}.yml
ln -s /etc/kolla/config/passwords-${OS_ENV}.yml /etc/kolla/passwords.yml

Links
https://docs.openstack.org/kolla-ansible/latest/user/operating-kolla.html

ipmitool

Install

sudo apt-add-repository universe
sudo apt install -y ipmitool
#sudo apt install -y freeipmi-tools

Install ipmitool (IPMI command line client)
[embed_url: https://raw.githubusercontent.com/panticz/installit/master/install.ipmitool.sh]

Configure BMC to DHCP

# show BMC interface informations
ipmitool lan print

# set dhcp IP
ipmitool lan set 1 ipsrc dhcp

# set static IP
ipmitool lan set 1 ipsrc static
ipmitool lan set 1 ipaddr 10.0.1.11
ipmitool lan set 1 netmask 255.255.255.0
ipmitool lan set 1 defgw ipaddr 10.0.1.252

# set VLAN
ipmitool -I lanplus -H ${BMC_IP} -U ${BMC_USER} -P ${BMC_PASS} ipmitool lan set 1 vlan id 1234
ipmitool -I lanplus -H ${BMC_IP} -U ${BMC_USER} -P ${BMC_PASS} ipmitool lan set 1 vlan id off

# soft restart BMC 
ipmitool -I lanplus -H ${BMC_IP} -U ${BMC_USER} -P ${BMC_PASS} mc reset warm

# hard restart BMC 
ipmitool -I lanplus -H ${BMC_IP} -U ${BMC_USER} -P ${BMC_PASS} mc reset cold

Commands

Supermicro

Firmware download
https://www.supermicro.com/support/resources/bios_ipmi.php

IPMI Supermicro management

# install
wget -q https://www.supermicro.com/wdl/utility/IPMICFG/IPMICFG_1.35.1_build.230912.zip -P /tmp/
unzip /tmp/IPMICFG_*.zip -d /tmp/
chmod +x /tmp/IPMICFG_*/Linux/64bit/IPMICFG-Linux.x86_64
/tmp/IPMICFG_*/Linux/64bit/IPMICFG-Linux.x86_64 -m
sudo cp /tmp/IPMICFG_*/Linux/64bit/IPMICFG-Linux.x86_64 /usr/local/sbin/ipmicfg

BIOS & IPMI Downloads
https://www.supermicro.com/support/resources/resource_links.cfm
https://www.supermicro.com/support/resources/bios_ipmi.php

sum (Supermicro Update Manager)
https://www.supermicro.com/SwDownload/SwSelect_Free.aspx?cat=SUM

consul 

echo '["abcdef123458"]' /var/consul/serf/local.keyring
service consul restart

# config
/etc/consul/config.json
/etc/consul/consul.d/*

# log
/var/log/syslog

CLI

consul catalog datacenters
consul catalog nodes
consul catalog services

consul monitor
consul validate /etc/consul/config.json 
consul operator raft list-peers

UI listen on external
https://stackoverflow.com/questions/35132687/how-to-access-externally-to-consul-ui

# Redirect UI to localhost
ssh -L 8500:localhost:8500 root@node1.example.com -N

# cat /etc/consul/config.json 
{
...
    "addresses": {
        "dns": "127.0.0.1",
        "grpc": "127.0.0.1",
        "http": "0.0.0.0",
        "https": "127.0.0.1"
    },
...

Remove service from specific node

ssh node1
consul services deregister -id=prometheus

Remove node
https://www.consul.io/commands/force-leave

consul force-leave com52-dev

checks
https://www.consul.io/api/agent/check.html

Get service state
https://consul.example.com/v1/agent/service/neutron

Icinga / Nagios checks

for SERVICE in $(consul catalog services); do
    echo ${SERVICE}
    /usr/lib/nagios/plugins/check_consul_service.py ${SERVICE}
    echo
done

Ansible module
https://docs.ansible.com/ansible/latest/modules/consul_module.html

Consul data directory

/var/consul/*

ESP
# remove service

Enable UEFI / PXE boot on Mellanox ConnectX NIC

Install Mellanox CLI tools (MFT)
http://www.mellanox.com/page/management_tools

#URL=https://www.mellanox.com/downloads/MFT/mft-4.15.0-104-x86_64-deb.tgz
URL=https://www.mellanox.com/downloads/MFT/mft-4.16.3-12-x86_64-deb.tgz

# install requirements
apt update
apt install -y gcc make dkms linux-headers-$(uname -r)

# download mft
wget -qO- ${URL} | tar xvz -C /tmp

# install mft
/tmp/mft-*-deb/install.sh

Enable UEFI and PXE boot

mst start
#for MST in $(ls /dev/mst/* | egrep -v '\.1'); do
for MST in $(ls /dev/mst/*); do
    echo ${MST}
    mlxconfig -d ${MST} -y set EXP_ROM_UEFI_x86_ENABLE=1
    mlxconfig -d ${MST} -y set EXP_ROM_PXE_ENABLE=1
    mlxconfig -d ${MST} q | egrep "EXP_ROM"
done

Debug: Show device state

mst status

for MST in $(ls /dev/mst/*); do
    mlxconfig -d ${MST} q
    echo
done

VMware: Installation Tools

Install open-vm-tools package

sudo apt-get install -y open-vm-tools
# Ubuntu Offline install
wget http://mirrors.kernel.org/ubuntu/pool/main/o/open-vm-tools/open-vm-tools_10.0.7-3227872-2ubuntu1_amd64.deb
wget http://mirrors.kernel.org/ubuntu/pool/main/libd/libdumbnet/libdumbnet1_1.12-7_amd64.deb
wget http://security.ubuntu.com/ubuntu/pool/main/libm/libmspack/libmspack0_0.5-1ubuntu0.16.04.3_amd64.deb
scp *.deb :/tmp/
ssh  dpkg -i /tmp/open-vm-tools_*_amd64.deb /tmp/libdumbnet*_amd64.deb /tmp/libmspack0_*_amd64.deb
# FreeBSD

pkg install open-vm-tools-nox11

# /etc/rc.conf

vmware_guest_vmblock_enable="YES"
vmware_guest_vmhgfs_enable="NO"
vmware_guest_vmmemctl_enable="YES"
vmware_guest_vmxnet_enable="YES"
vmware_guestd_enable="YES"

/usr/local/etc/rc.d/vmware-guestd enabled
/usr/local/etc/rc.d/vmware-guestd start

VMware image (when open-vm-tools not available)
https://packages.vmware.com/tools/index.html

# Actions > Guest OS > Upgrade VMware Tools > Interactive Upgrade > Upgrade
mount /dev/cdrom /mnt
tar -C /tmp -xzf /mnt/VMwareTools-*.tar.gz
umount /mnt

/tmp/vmware-tools-distrib/vmware-install.pl -default

rm -rf /tmp/vmware-tools-distrib
# FreeBSD

https://my.vmware.com/de/web/vmware/details?downloadGroup=VMTOOLS1015&productId=614

GitLab: Backup to S3

Configure Git

# /etc/gitlab/gitlab.rb
gitlab_rails['backup_upload_connection'] = {
  'provider' => 'AWS',
  'region' => 'foo-west-1',
  'aws_access_key_id' => 'KEY123',
  'aws_secret_access_key' => 'PASS124',
  'endpoint' => 'https://s3.example.com'
}
gitlab_rails['backup_upload_remote_directory'] = 'backups'
gitlab_rails['backup_keep_time'] = 604800

gitlab-ctl reconfigure

Test

gitlab-rake gitlab:backup:create

Configure periodic backup

# crontab -e
0 2 * * * /opt/gitlab/bin/gitlab-rake gitlab:backup:create CRON=1

Links
https://docs.gitlab.com/ce/raketasks/backup_restore.html#uploading-backups-to-a-remote-cloud-storage

GitLab: LFS on S3

Configure GitLab

...
gitlab_rails['lfs_enabled'] = true
gitlab_rails['lfs_object_store_enabled'] = true
gitlab_rails['lfs_object_store_proxy_download'] = true
gitlab_rails['lfs_object_store_remote_directory'] = "lfs"
gitlab_rails['lfs_object_store_connection'] = {
   'provider' => 'AWS',
   'aws_access_key_id' => 'KEY123',
   'aws_secret_access_key' => 'PASS1234',
   'endpoint' => 'https://s3.example.com',
}

gitlab-ctl reconfigure

Install (on client)

apt-get install -y git git-lfs

# create LFS testfile
head -c 1M /dev/urandom > file2.bin

Add LFS file to repository

git lfs install
git lfs track *.bin
# git lfs track "*.bin"
# git lfs track images/
git add .gitattributes *.bin
git commit -m "Commit LFS file"
git push

Fix me
LFS file download / checkout broken

Debug

git lfs ls-files
git lfs env

Links
https://about.gitlab.com/2017/01/30/getting-started-with-git-lfs-tutorial/
http://blog.testdouble.com/posts/2017-01-25-how-and-why-to-use-git-lfs

GitLab: Artifacts on S3

Create bucket

s3cmd mb s3://artifacts

Configure GitLab

/etc/gitlab/gitlab.rb
...
nginx["client_max_body_size"] = "1024m"
gitlab_rails["artifacts_enabled"] = true
gitlab_rails["artifacts_object_store_enabled"] = true
gitlab_rails["artifacts_object_store_remote_directory"] = "artifacts"
gitlab_rails["artifacts_object_store_connection"] = {
  "provider" => "AWS",
  "region" => "us-west-1",
  "aws_access_key_id" => "KEY1234",
  "aws_secret_access_key" => "PASS1234",
  "endpoint" => "https://s3.example.com"
}

gitlab-ctl reconfigure

Links
https://docs.gitlab.com/ee/ci/yaml/#artifacts
https://docs.gitlab.com/ee/administration/job_artifacts.html#using-object-storage