Home | panticz.de

Elasticsearch: curator

Install
https://www.elastic.co/guide/en/elasticsearch/client/curator/5.x/apt-repository.html

wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb [arch=amd64] https://packages.elastic.co/curator/5/debian stable main" > /etc/apt/sources.list.d/curator.list
sudo apt-get update && sudo apt-get install elasticsearch-curator

# sudo apt install -y elasticsearch-curator
# pip install elasticsearch-curator

Show usage

curl elasticsearch.example.com:9200/_cat/indices

curator.yml

---
client:
  hosts:
    - elasticsearch.example.com
  port: 9200
  url_prefix:
  use_ssl: False
  certificate:
  client_cert:
  client_key:
  ssl_no_validate: False
  http_auth:
  timeout: 30
  master_only: False

logging:
  loglevel: INFO
  logfile:
  logformat: default
  blacklist: ['elasticsearch', 'urllib3']

delete_indices.yml

---
actions:
  1:
    action: delete_indices
    description: >-
      Delete indices older than 14 days
    options:
      ignore_empty_list: True
      timeout_override:
      continue_if_exception: False
      disable_action: False
    filters:
#    - filtertype: pattern
#      kind: prefix
#      value: logstash-
#      #value: flog-
#      exclude:
#    - filtertype: pattern
#      kind: prefix
#      value: flog-
#      exclude:
    - filtertype: age
      source: name
      direction: older
      timestring: '%Y.%m.%d'
      unit: days
      unit_count: 7
      exclude:

Delete old data

curator --config ./curator.yml ./delete_indices.yml #--dry-run

Links
http://www.madhur.co.in/blog/2017/04/09/usingcuratordeleteelasticindex.html
https://cezarypiatek.github.io/post/scheduled-elasticsearch-cleanup/

OpenStack: Designate (DNSaaS)

Configure kolla-ansible

#echo 'designate_tag: "7.0.1.2"' >> /etc/kolla/globals.yml
sed -i 's/[#]dns_interface:/dns_interface:/g' /etc/kolla/globals.yml
sed -i 's/[#]enable_designate: .*/enable_designate: "yes"/g' /etc/kolla/globals.yml
sed -i 's/[#]enable_horizon_designate:/enable_horizon_designate:/g' /etc/kolla/globals.yml
sed -i 's/[#]designate_ns_record: .*/designate_ns_record: "pool.{{ os_environment }}.example.com"/g' /etc/kolla/globals.yml

Configure pool

mkdir -p /etc/kolla/config/foo/designate

/etc/kolla/config/foo/designate/pools.yaml
- name: default-bind
  description: Default BIND9 Pool
  attributes: {}
  ns_records:
    - hostname: ns1.dev.i.example.com.
      priority: 1
  nameservers:
    - host: 10.0.4.45
      port: 53
  targets:
    - type: bind9
      description: BIND9 Server 1
      masters:
        - host: 10.0.4.135
          port: 5354
      options:
        host: 10.0.4.45
        port: 53
        rndc_host: 10.0.4.45
        rndc_port: 953
        rndc_key_file: /etc/designate/rndc.key

Deploy Designate container

kolla-ansible -i /etc/kolla/inventory deploy -t designate

# Test endpoint
curl designate.service.${ENV}.i.example.com:9001

Configure neutron

grep designate_keystone_password /etc/kolla/passwords.yml 

vi /etc/kolla/config/ew/neutron.conf
[DEFAULT]
...
external_dns_driver = designate
dns_domain = pool.stage.example.com.

[designate]
url = http://designate.service.stage.i.example.com:9001/v2
auth_type = password
auth_url = http://keystone-admin.service.stage.i.example.com:35357
username = designate
password = xxxxxxxxxxxxxxxxxxx
project_name = service
project_domain_name = default
user_domain_name = default
allow_reverse_dns_lookup = True
ipv4_ptr_zone_prefix_size = 24
ipv6_ptr_zone_prefix_size = 116
ptr_zone_email = foo@example.com

kolla-ansible -i /etc/kolla/inventory reconfigure -t neutron

Install designate client

Lego: Let’s Encrypt client and ACME library

Get LEGO binary
https://github.com/go-acme/lego/releases/

wget  https://github.com/go-acme/lego/releases/download/v5.2.2/lego_v5.2.2_linux_amd64.tar.gz
tar xzf lego_v5.2.2_linux_amd64.tar.gz

Show certificates

ll ~/.lego/certificates/

OpenStack Designate

# test with designage
/tmp/lego --accept-tos --email="foo@bar.com" --domains="*.bar.com" --dns designate run
... designate: some credentials information are missing: OS_AUTH_URL,OS_USERNAME,OS_PASSWORD,OS_TENANT_NAME,OS_REGION_NAME

# stop service on port 80 / 443
service nginx stop
service apache2 stop

# Request certificate
/tmp/lego --accept-tos --email="foo@bar.com" --domains="bar.com" --http run

# Request wildcard certificate
/tmp/lego --accept-tos --email="foo@bar.com" --domains="bar.com" --domains="*.bar.com" --dns manual run

# restart service on port 80 / 443
service nginx start
service apache2 start

Request wildcart certificate

./lego \
    --server https://acme-v02.api.letsencrypt.org/directory \
    --accept-tos=true \
    --dns.resolvers="ns1.example.com:53" \
    --dns.resolvers="ns2.example.com:53" \
    --email="info@example.com" \
    --dns=nictool \
    --key-type=rsa2048 \
    --pem \
    --domains="*.${DOMAIN}" \
    run

Renew certificates
https://go-acme.github.io/lego/usage/cli/examples/

LXD: storage

Configure default storage

lxc storage create zfs zfs source=rpool/lxd
lxc profile device add default root disk path=/ pool=zfs

lxc storage list
lxc storage delete default

# zfs
lxc storage create zfs zfs source=tank/lxd
lxc storage list

# delete default storage
lxc storage volume list default
lxc storage volume delete default image/7d788819a5a97433db8470ee68370ec69e829b429800fa28b5524f0411490ce9
lxc storage delete default

# move container to another storage
CONTAINER=www1
lxc move ${CONTAINER} ${CONTAINER}-tmp -s nvme
lxc move ${CONTAINER}-tmp ${CONTAINER}
lxc start ${CONTAINER}

Configure ZFS storage

lxc profile device del dev-zfs root
lxc profile device add dev-zfs root disk path=/ pool=zfs

Storage

# change container storage quota
lxc config device set  root size 100GB

# lvm thin pool
lvcreate -L 250G --thinpool kvm system
lxc storage create kvm lvm source=system lvm.thinpool_name=kvm

# unix-block
lxc config device add c1 xvdb1 unix-block source=/dev/xvdb1 required=false
lxc config device remove gitlab-runner3-dev xvdb2

Container root

/var/lib/lxd/storage-pools/default/containers/*/rootfs/

Links
https://github.com/lxc/lxd/blob/master/doc/storage.md
https://linuxcontainers.org/lxd/docs/master/storage

GitLab: Docker CI pipeline

Optinal: Create nested LXD container
http://www.panticz.de/lxd/nesting

CONTAINER_NAME=gitlab-runner1-dev
lxc launch ubuntu:18.04 ${CONTAINER_NAME} -p disk-zfs -p nic-dev-mgmt -c boot.autostart=true -c security.nesting=true -c security.privileged=true
#-c volatile.dev-mgmt.hwaddr=00:11:22:33:44:55

lxc exec ${CONTAINER_NAME} -- apt update
lxc exec ${CONTAINER_NAME} -- apt dist-upgrade
lxc exec ${CONTAINER_NAME} -- apt purge -y lxd lxd-client snapd unattended-upgrades
lxc exec ${CONTAINER_NAME} -- apt autoremove

lxc file push /root/.ssh/authorized_keys ${CONTAINER_NAME}/root/.ssh/authorized_keys
lxc exec ${CONTAINER_NAME} -- bash -c "sed -i 's/eth0:/dev-mgmt:/g' /etc/netplan/50-cloud-init.yaml"
lxc exec ${CONTAINER_NAME} -- netplan apply

printf 'lxc.apparmor.profile = unconfined\nlxc.cgroup.devices.allow = a\nlxc.mount.auto=proc:rw sys:rw\nlxc.cap.drop=' | lxc config set ${CONTAINER_NAME} raw.lxc -
lxc restart ${CONTAINER_NAME}

Install Docker inside LXD container
# http://www.panticz.de/install-docker

apt update
sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
sudo apt-get install -y docker-ce

Install GitLab-runner (as Docker container)
https://docs.gitlab.com/runner/install/docker.html

docker run -d \
  --name gitlab-runner \
  --restart always \
  -v /srv/gitlab-runner/config:/etc/gitlab-runner \
  -v /var/run/docker.sock:/var/run/docker.sock \
  gitlab/gitlab-runner:latest

Configue GitLab-runner

jitsi

Install
https://desktop.jitsi.org/Main/DebianRepository.html

wget -qO - https://download.jitsi.org/jitsi-key.gpg.key | sudo apt-key add -
sudo sh -c "echo 'deb https://download.jitsi.org stable/' > /etc/apt/sources.list.d/jitsi-stable.list"
sudo apt-get -y update
sudo apt-get -y install jitsi

OpenStack: Recreate neutron port

Get broken / INACTIVE ports

DB_NAME=neutron
DB_USER=neutron
DB_PASS=$(grep neutron_database_password /etc/kolla/passwords.yml | cut -d " " -f2)
mysql --host=${DB_HOST} --password=${DB_PASS} --port=6033 --user=${DB_USER} --database=${DB_NAME}

select port_id, host from ml2_port_bindings where status = 'INACTIVE';
select port_id, host from ml2_port_bindings where status = 'INACTIVE' and host = 'com3-dev';
select * from ml2_port_bindings where port_id = "65db0c73-1533-4c36-9fd4-79b5e4e9704c";
#select * from ml2_port_binding_levels where port_id = "65db0c73-1533-4c36-9fd4-79b5e4e9704c" \G

# debug
PORT_IDS=$(mysql --host=${DB_HOST} --password=${DB_PASS} --port=6033 --user=${DB_USER} --database=${DB_NAME} \
  -se "select port_id from ml2_port_bindings where status = 'INACTIVE'")
echo ${PORT_IDS}

# get VMs
for PORT_ID in ${PORT_IDS}; do
    echo ${PORT_ID}
    openstack server show -c name -c "OS-EXT-SRV-ATTR:host" $(openstack port show -c device_id -f value ${PORT_ID})
done

Recreate port

SERVER_ID=f4b8828d-a717-49f6-8ad3-7b0bb1921123

nova interface-list ${SERVER_ID}

PORT_ID=57944199-014c-4907-9d6e-1fb233151234

# debug
openstack port show ${PORT_ID} -c name -c network_id -c mac_address -c fixed_ips -c security_group_ids -c project_id -c device_id

get port data

FIX: While synchronizing instance power states, found XX instances in the database and XX instances on the hypervisor

Check kibana logs for
"While synchronizing instance power states, found"

Check libvirt VMs vs. nova DB

COMPUTE_NODE=com1-dev

# get shutoff VMs on compute node
VMS_COMPUTE=$(ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh list --state-shutoff --uuid | sed '/^$/d' | sort)
# echo "${VMS_COMPUTE}"

# get shutoff VMs from nova DB
VMS_NOVA=$(ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server list --all --host ${COMPUTE_NODE} -c ID -f value --status SHUTOFF" | sort)
# echo "${VMS_NOVA}"

# diff shutoff VMs
comm -3 <(echo "${VMS_COMPUTE}") <(echo "${VMS_NOVA}")

Remove shutdown VM from libvirt

VM_ID=c3fbcc6b-2dfe-4aa1-82bd-522b161a37a9

# OPTINAL: backup VM xml config
# ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dumpxml ${VM_ID} > /tmp/${COMPUTE_NODE}.${VM_ID}.xml

# show VM details (verfiy that OS-EXT-SRV-ATTR:host is not ${VMS_COMPUTE})
ssh os-admin-dev "source /etc/kolla/admin-openrc.sh; openstack server show -c OS-EXT-SRV-ATTR:host -c name -c status ${VM_ID}"

# show VM details on compute node (verify that State: is "shut off")
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh dominfo ${VM_ID}

# remove unnecessary VM
ssh ${COMPUTE_NODE} docker exec -i nova_libvirt virsh undefine ${VM_ID}

OpenVswitch: cleanup interfaces without tap device

Show bridges without tab device

brctl show | egrep "qvb|tap" | sed '$!N;/\n.*tap/d;P;D' | awk '{print substr($1,4,8)}'

Get interface details

TOKEN=cac559da

# show port details
docker exec openvswitch_vswitchd ovsdb-client dump | grep ${TOKEN}

# get OVS port
docker exec openvswitch_vswitchd ovs-vsctl list-ports br-int | grep ${TOKEN}

# get OVS interface
docker exec openvswitch_vswitchd ovs-vsctl list-ifaces br-int | grep ${TOKEN}

# show host bridges
brctl show | grep ${TOKEN}

# show host interfaces
ip a | grep ${TOKEN}

Delete interface

PORT_TOKEN=a101dd83

# get port ID
PORT_ID=$(docker exec openvswitch_vswitchd ovs-vsctl list-ports br-int | grep ${PORT_TOKEN})
echo ${PORT_ID}

# remove port
docker exec openvswitch_vswitchd ovs-vsctl del-port br-int ${PORT_ID}

# remove bridge
BRIDGE_ID=${PORT_ID/qvo/qbr}
echo ${BRIDGE_ID}
ip link set ${BRIDGE_ID} down
brctl delbr ${BRIDGE_ID}

# remove interface
ip link delete ${PORT_ID}

ddrescue

Install

apt install -y gddrescue
dpkg -i gddrescue_1.22-1_amd64.deb

Rescue

ddrescue /dev/sda sda.dd sda.ddrescue.log