Move Elasticsearch data to dedicated LV 

# create lvm
pvcreate /dev/disk/by-id/ata-INTEL_SSDSC2KB076T8_*
vgcreate data /dev/disk/by-id/ata-INTEL_SSDSC2KB076T8_*
lvcreate --name elasticsearch --size 2T data
mkfs.ext4 /dev/data/elasticsearch

# pre-sync data
mount /dev/data/elasticsearch /mnt/
rsync -aHAXx --numeric-ids /var/lib/docker/volumes/elasticsearch/ /mnt/

# sync data
docker stop elasticsearch
rsync --delete -aHAXxv --numeric-ids /var/lib/docker/volumes/elasticsearch/ /mnt/
rsync --delete -aHAXxv --numeric-ids /var/lib/docker/volumes/elasticsearch/ /mnt/
umount /mnt

# mount new LV
mv /var/lib/docker/volumes/elasticsearch{,.org}
mkdir /var/lib/docker/volumes/elasticsearch
echo "/dev/mapper/data-elasticsearch   /var/lib/docker/volumes/elasticsearch    ext4    defaults    0    2" >> /etc/fstab 
mount -a

# restart elasticsearch container
docker start elasticsearch
# docker logs elasticsearch

# visudo -f /etc/sudoers.d/nagios
echo "nagios ALL=(root) NOPASSWD: /usr/lib/nagios/plugins/check_disk *" >> /etc/sudoers.d/nagios

# allow NRPE access to volume
vi /etc/nagios/nrpe.cfg
command[check_disk]=/usr/bin/sudo /usr/lib/nagios/plugins/check_disk -w "10%" -c "5%" -l -X devtmpfs -X sysfs -X tmpfs -X cgroup -X proc -X udev -X debugfs -X overlay -X cgmfs -X tracefs -A -I lxc -I nsfs
service nagios-nrpe-server restart

# Debug
journalctl _COMM=sudo -f

Elasticsearch: curator

Install
https://www.elastic.co/guide/en/elasticsearch/client/curator/5.x/apt-repository.html

wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb [arch=amd64] https://packages.elastic.co/curator/5/debian stable main" > /etc/apt/sources.list.d/curator.list
sudo apt-get update && sudo apt-get install elasticsearch-curator

# sudo apt install -y elasticsearch-curator
# pip install elasticsearch-curator

Show usage

curl elasticsearch.example.com:9200/_cat/indices

curator.yml

---
client:
  hosts:
    - elasticsearch.example.com
  port: 9200
  url_prefix:
  use_ssl: False
  certificate:
  client_cert:
  client_key:
  ssl_no_validate: False
  http_auth:
  timeout: 30
  master_only: False

logging:
  loglevel: INFO
  logfile:
  logformat: default
  blacklist: ['elasticsearch', 'urllib3']

delete_indices.yml

---
actions:
  1:
    action: delete_indices
    description: >-
      Delete indices older than 14 days
    options:
      ignore_empty_list: True
      timeout_override:
      continue_if_exception: False
      disable_action: False
    filters:
#    - filtertype: pattern
#      kind: prefix
#      value: logstash-
#      #value: flog-
#      exclude:
#    - filtertype: pattern
#      kind: prefix
#      value: flog-
#      exclude:
    - filtertype: age
      source: name
      direction: older
      timestring: '%Y.%m.%d'
      unit: days
      unit_count: 7
      exclude:

Delete old data

curator --config ./curator.yml ./delete_indices.yml #--dry-run

Links
http://www.madhur.co.in/blog/2017/04/09/usingcuratordeleteelasticindex.html
https://cezarypiatek.github.io/post/scheduled-elasticsearch-cleanup/

elasticsearch