HowTo | panticz.de

bash
linux

http://www.panticz.de/find
http://www.panticz.de/rsync
http://www.panticz.de/sed
http://www.panticz.de/ssh
http://www.panticz.de/ip
http://www.panticz.de/apt
http://www.panticz.de/hardware

[embed_url: https://raw.githubusercontent.com/panticz/scripts/master/howto.sh]

# sudo over SSH
echo "ls -l /root/.ssh/id_rsa*" | ssh host sudo bash
ssh -t host "sudo -s bash -c \"ls -l /root/.ssh/id_rsa*\""

# create bakup file
cp path/to/file{,.$(date -I)}

# clear cache
sudo sh -c 'echo 1 >/proc/sys/vm/drop_caches'
sudo sh -c 'echo 2 >/proc/sys/vm/drop_caches'
sudo sh -c 'echo 3 >/proc/sys/vm/drop_caches'

# wget with authentification
wget -q --user=foo --password=bar http://www.example.com -O -

# reduce swappiness
echo "vm.swappiness = 1" > /etc/sysctl.d/90-swap.conf

# listing swap usage by process
grep VmSwap /proc/*/status | grep -v " 0 kB"

for file in /proc/*/status ; do
    awk '/VmSwap|Name/{printf $2 " " $3}END{ print ""}' $file;
done | grep -E 'kB|mB' | sort -k2 -V

# parse jeson keys
cat /tmp/in.json | jshon -k
cat /tmp/in.json | jq -r 'keys[]'

# set date
date +%Y%m%d -s "20150430"

# date with year-mounth-day_hour:minute:second
date +%F_%H:%M:%S

# calculate quarter from calendar week
date -d "${YEAR}-01-01 +${MONTH} week" "+%q"

# filter
echo $file | sed 's/[^0-9]*//g' 
echo "${file//[!0-9]/}"

# use HTTP/HTTPS proxy from command line
export http_proxy=http://proxy.example.com:3128 wget http://www.example.com -O -
curl -H -x http://proxy.example.com:3128 'https://www.example.com/paht/index.php'

# get lines beginning with second line
cat /path/to/file | tail -n +2

# get total disk usage
df -h --total

# diff between remote files (over SSH)
diff <(ssh srv1.example.com cat /etc/fstab) <(ssh srv2.example.com cat /etc/fstab)

# /etc/fstab entry non failing / blocking
/media/usb_backup_1   /home/user1/backup   none   nofail,defaults,bind   0   0


# check platform
if [[ $(getconf LONG_BIT) = "64" ]]; then
    echo "64bit"
else
    echo "32bit"
fi

# tar
tar -T input_names.txt -cjf /path/to/archive.tar.bz2

tar -cvjf ansible.$(date -I).tar.bz2 --exclude='*.zip' --exclude=.git ansible

# extract xz file on the fly
wget -q https://www.example.com/file.tar.xz -O - | tar -C /tmp/ -Jx

# remove metadata from PDF
exiftool -all= inputfile,pdf

# reduce pdf
gs -dNOPAUSE -dBATCH -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/ebook -sOutputFile=out.pdf in1.pdf in2.pdf

# preserve links
gs -q -dNOPAUSE -dBATCH -sDEVICE=pdfwrite -sPAPERSIZE=a4 -dPDFSETTINGS=/ebook -dPDFFitPage -dPrinted=false -sOutputFile=OUT.pdf IN.pdf

# search for string in PDFs recursive
find . -type f -iname "*.pdf" -print0 | while IFS= read -r -d '' file; do
    pdftotext "$file" - 2>/dev/null | grep -Hn "foo"
done

# show current runlevel
who -r

# remove dmraid
sudo dmraid -x 
sudo dmraid -rE

# show user member groups
groups 

# delete user from group
sudo deluser  sudo

# TextMe:
openssl s_client -showcerts -connect www.example.com:443 > /tmp/cacert.pem
curl --cacert /tmp/cacert.pem https://mail.example.com/

# convert Ogg to MP3
sudo apt-get install ffmpeg
IFS=$(echo -en "\n\b")
for i in $(find -name "*.ogg"); do
    ffmpeg -i ${i} -c:a libmp3lame -ab 160k -map_metadata 0:s:0 "${i/ogg/mp3}"
done

# resice images
for i in *.jpg; do
    convert $i -pointsize 72 -quality 90 -resize 512x384 -unsharp 0x.5 ${i/.jpg/_small.jpg}
done

# restart / reload network configuration
sudo ifdown --exclude=lo -a && sudo ifup --exclude=lo -a



# cat file starting from 6 line
cat /tmp/file | tail -n +6

# list established connections
sudo netstat -nap | grep EST

# cat with ecsape
cat <<'EOF'> /tmp/testfile
test $foo
EOF

# check HTML header
curl --head www.example.com

# send header
curl -vs -H "Host: example.com" "http://www.foo.com" >/dev/null


# test memcached
telnet memcached.example.com 11211
stats
quit

# TeamViewer QuickSupport on-the-fly (download and start)
wget -q http://download.teamviewer.com/download/teamviewer_qs.tar.gz -O - | tar -C /tmp/ -xz && /tmp/teamviewerqs/teamviewer

# get network range
whois $(dig +short whois-servers.net | tail -1) | grep NetRange

# list wifi device settings
rfkill list all
echo 0 > /sys/class/rfkill/rfkill0/hard

# configure NIC speed
apt-get install -y ethtool
ethtool -s eth1 speed 1000 duplex full

# get NIC hardware mac address
ethtool -P eth0

# UDP on iperf server
iperf -s [-u]

# UDP on ipfer client
iperf -c 192.168.254.1 -t 600 [-u] [-B 192.168.1.1]

# set hostname
sudo hostnamectl set-hostname new-hostname

# dmesg follow and human readable
dmesg -wH

# print current runlevel
who -r
runlevel

# erase a DVD-RW via command line
umount /dev/sr0
wodim dev=/dev/sr0 blank=fast

# sort directory by size
du -sh * | sort -h
du -h | grep "^[0-9,]*[MGT]" | sort -h

# sort by nuber (human readable)
sort -V
sort --version-sort

# look screen from command line
gnome-screensaver-command -l

# mount filesystems bevore chroot environment
mount /dev/sda1 /mnt
cd /mnt
mount -t proc proc proc/
mount -t sysfs sys sys/
mount -o bind /dev dev/
chroot /mnt

# recovery mode kernel boot parameter
fsck.mode=force
fsck.repair=yes
break

# get count of running processes
pgrep -cf 

# monitor / restart service when not running
pgrep -f glassfish 1>/dev/null || service glassfish restart

# format output with awk
cat file.csv | awk -v OFS="\t" -F ";" '{printf "%s\t%s\t%s\t%.0f\t%8.0f\t\n", $1, $4, $2, $7/1024, $6}'

# start software raid
mdadm --assemble --scan -v

# install grub2
grub-install --no-floppy --root-directory=/mnt /dev/sda

Bash redirection
# stdout to file
programm > out.log

# stderr to file
programm 2> out.err

# stdout and stderr to file
programm &> out.log

# stdout to stderr
programm 1>&2

# stderr to stdout
programm 2>&1

# resize dos partition / bootdisk
fatresize -s 33M bios.img
dd if=bios.img of=bios_34m.img bs=1M count=34

# set systemwide proxy
echo "export http_proxy=http://proxy.example.com:3128" >> /etc/bashrc

# crop PDF
gs -o out.pdf -sDEVICE=pdfwrite -c "[/CropBox [20 20 80 1000] /PAGES pdfmark" -f in.pdf

# remove route and blank lines
cat  | grep -v '#' | grep '\S

# composer install (create vendor directory)
./composer install

# screen scroll UP
CTRL + a
ESC
UP

# reconnect screen
sudo screen -list
sudo screen -d -r root/2953.ubuntu-release-upgrade-screen-window

# show processes with high cpu / memory usage
ps -eo pid,ppid,cmd,%mem,%cpu --sort=%cpu --no-headers
top -b -o +%CPU | head

ps -eo size,pid,user,command --sort -size | awk '{ hr=$1/1024 ; printf("%13.2f Mb ",hr) } { for ( x=4 ; x<=NF ; x++ ) { printf("%s ",$x) } print "" }' |    cut -d "" -f2 | cut -d "-" -f1

# sub shell
( (sleep 10; date) > /tmp/q ) &

# create user
sudo useradd user1 -m -s /bin/bash
sudo usermod -a -G sudo user1

sudo adduser user2 --shell /bin/bash --group sudo

# add user to group sudo
sudo usermod -a -G sudo 

# check backup
sudo -u git bash -c "find /var/opt/gitlab/backups -name *_$(date +%Y_%m_%d)_gitlab_backup.tar -size +50M"

# ntp
apt-get install -y ntp
cat /etc/ntp.conf
ntpd -q -g
sudo ntpdate -u ntp.ubuntu.com 
hwclock --systohc
ntpq -p

# identify network device
sudo nmap -O -v 10.0.0.5

# reconnect to process / bash
reptyr 

# KeePassX command
cmd://bash /foo/bar.sh {USERNAME} {PASSWORD}

# mount samba share
sudo apt-get install -y cifs-utils
sudo mount -t cifs -o user=foo //backup.example.cmo/bar /mnt

# remove LSI / intel software raid
wipefs -a /dev/sda

# configure camera codec
apt-get install -y v4l-utils 
v4l2-ctl --list-formats
v4l2-ctl --set-fmt-video=width=640,height=480,pixelformat=0

# benchmark disc
bwm-ng -i disk

# reduce disk I/O
echo 1 > /proc/sys/vm/dirty_background_ratio
echo 80 > /proc/sys/vm/dirty_ratio

# renice rsync
sudo pgrep rsync | xargs ionice -c3 -p
sudo pgrep rsync | xargs renice -n 19 -p

# date
echo $(date +%Y-%m-%d\ %H:%M:%S)

# reboot with delay
shutdown -r +5 "Maintaince: Server will restart in 5 minutes."

# partitionize whole disk from command line
parted /dev/sdc -s mklabel msdos
parted /dev/sdc -s mkpart primary 2048s 100%
mkfs.vfat -n USB_DISK /dev/sdc1

# get MAC address for a interface
cat /sys/class/net/eth0/address

# Apache: Top file not found
cat /var/log/apache2/error.log | grep "File does not exist" | cut -d " " -f 13 | sort | uniq -c | sort -rn | egrep -v '.*1 '

# list physical disc usage
df -hT | sed -n '1p;/^\//p;'

# replace text recursively
find -type f -size -50k -exec sed -i 's|foo|bar|g' {} \;

# find and del dirs smaller then (not tested)
find . -maxdepth 1 -type d | 
    while read dir; do [ $(du -s "$dir") -le 102400 ] && rm -f "$dir"; done

# enable cron logging to dedicated file on ubuntu
sed -i 's|#cron|cron|g' /etc/rsyslog.d/50-default.conf
service rsyslog restart

# last
last -x reboot
last -x shutdown

# rename files in directory
rename 's/out/txt/' *.out

# rename file
mv /tmp/test{,.off}

# search in pdfs
find . -name '*.pdf' -exec sh -c 'pdftotext "{}" - | grep --with-filename --label="{}" --color "foo"' \;

# findest oldest access directory
find -type d -printf '%T+ %p\n' -not \( -path "./exclude_dir/*" -type d  -prune \) | sort | head -n 20

# create iso from directory content
genisoimage -R -o ../foo.iso *

# df formated output
df -h --output=size,used,avail,pcent /home

# rescan disk
echo "1" > /sys/class/block/sdX/device/rescan

# add user to group and reload group without logout
sudo usermod -a -G lxd ${USER}
su - ${USER}

# linux hard reset
echo s > /proc/sysrq-trigger
echo b > /proc/sysrq-trigger

# ext4 space reserved for root
tune2fs -l /dev/sdb1 | grep -i reserved
sudo tune2fs -m 0 /dev/sdb1

# get size for hidden directories
du -sh .{a..z}* 2>/dev/null | grep G

# convert textfile to UTF-8
iconv -f ISO-8859-1 IN.txt -t UTF-8 -o OUT.txt

# process uptime
ls -al /proc | grep 2719

# kill all apache
kill -9 $(pidof apache2)
pkill -9 apache2

# convert string to lower case
tr A-Z a-z < file_in > file_out

# list zip content
unzip -l archive.zip 

# zip: max compress, move file, ignore directory structure
DUMP=$(hostname).${DB}.$(date -I).sql
zip -9qmj ${DUMP%.2*}.sql.zip /tmp/${DUMP}

# parallel compress (keep the old file)
pbzip2 -k9 file1.in file2.in ...

# extract from URL
wget https://example.com/file.iso.bz2 -qO- | bzip2 -d > /path/file_out.iso

# yesterdays date
date -d yesterday +%Y-%m-%d

# show openend connections
netstat -tn grep :80

# disable root password
sudo passwd -l root

# inotifywait conf watch (single line)
while true; do inotifywait --quiet --monitor --event create,delete --exclude "[^c][^o][^n][^f]$" /tmp | /usr/sbin/nginx -t && /usr/sbin/service nginx reload; done

# diff loop
DIR=/etc/libvirt/qemu
for FILE in $(ls -1 ${DIR}/*.xml); do
    CMD="diff ${FILE} ${FILE}.$(date -I)"
    echo "${CMD}"
    ${CMD}
done

# diff exclude
diff --exclude=direxclude -r dir1 dir2

# display list of system calls
strace 

# replace duplicates with hardlinks
rdfind -makehardlinks true .

# show user default shell
getent passwd ${LOGNAME} | cut -d: -f7

# change disk label
# FAT
sudo fatlabel /dev/sdxN my_disk_1

# exFAT:
sudo exfatlabel /dev/sdxN my_disk_1

# NTFS
sudo ntfslabel /dev/sdxN my_disk_1

# ext2/3/4
sudo e2label /dev/sdxN my_disk_1

# BTRFS
sudo btrfs filesystem label /dev/sdxN my_disk_1

# get external / public IP
curl ifconfig.me
curl ipinfo.io/ip
curl 116.202.55.106

# check file size
[[ $(find /media/backup/ -size +100M -name bkp_$(date -I)_*.dmp.bz2 ) ]] && echo true || echo false

# find mime-type for a file
file --mime-type file.sh

# change date
touch -d 20180101 file1

# kill all defunct
ps -ef | grep defunct | grep -v grep | cut -b8-20 | xargs kill -9

# reload profile without relogin
su - $USER

# get file look
sudo fuser -v /var/cache/debconf/config.dat

# get linked libraries
ldd -d -r $(which wget)

# remove string from string
echo "my foo bar" | sed -e s/foo//

# copy output to one line
your_command | paste -sd, -

# remove last liste from file
cat /tmp/file.in | sed '$ d' > /tmp/file.out

# sync time
systemctl stop ntpd
ntpdate ntp.ubuntu.com
systemctl start ntpd
systemctl enable ntpd
systemctl status ntpd

# generate mac address
openssl rand -hex 6 | sed 's/\(..\)/\1:/g; s/:$//'

# get script path
echo $(dirname $(readlink -f "$0"))

# create 7zip encrypted backup
7z a  -p -mhe=on '-xr!*.jpg' '-xr!OLD' ~/backup/bkp.$(date -I).7z ~/dir1/* ~/.ssh

# print duplicate lines
cat FILE| uniq -d

# remount root read only
mount -f -o remount,ro /

# Remove all characters except of alphanumeric and "-":
echo "a b-1_2" | sed "s/[^[:alnum:]-]//g"

# Print unicode char
ctrl + shift + u 
2665
enter

# remount / force read only mount of root filesystem
umount -a
echo u > /proc/sysrq-trigger
mount -f -o remount,ro /dev/sda1

# change filesystem UUID
tune2fs -U 2efc22cb-98a4-448d-8fbe-a895462703c0 /dev/sdc1

# force XFS filesystem check
force fsck.xfs to run xfs_repair at boot time by creating a /forcefsck file or booting the system with "fsck.mode=force" on the kernel command line

# set initcwnd
ip route change default via 10.0.1.254 dev eth0 onlink initcwnd 32

# join three files
join -e- -a1 -a2 -o 0 1.2 2.2 file1 file2 | join -e- -a1 -a2 -o 0 1.2 1.3 2.2 - file3 | column -t

Sequence
https://linuxhint.com/bash_range/

# get sequence every 5
echo {50..100..5}

# decrementing sequence
seq 10 -2 1

# leading zeros
seq -w 1 12

# separator
seq -s"," 10

# formated
seq -f "%f" 3 0.8 6
seq -f "2019-01-%02g" 31

# multiple sequence
echo www{{1..3},{10..13}}-{dev,stage}

# debug dhcp
dhcping -s 10.0.0.253 -h fa:11:22:33:44:55

# configure keyboard language
loadkeys de

# umount luks
dmsetup remove /dev/mapper/luks-xxx

# get NVMe data
nvme list --output-format=json | jq -r .Devices[].ModelNumber

# umount recursively
umount  /mnt -R

# show reboots
last | egrep "down|boot"

# check trim support (when DISC-MAX != 0B)
lsblk -D /dev/sda | grep 2B

# dd with progress
dd if=/dev/sdc of=/dev/sdd bs=4096 status=progress

# sum lines together
paste -s -d+ infile | bc

# force root filesystem check
echo 1 >/sys/block/sda/sda1/ro
mount -f -o remount,ro /mount/point
fsck.ext4 /dev/sda1

# truncate file
truncate --size=1G out.log

# write to file as root
cat << EOF | sudo tee /etc/file.conf
...
EOF

# directory to iso
genisoimage -o output_image.iso directory_name

# show host IP addresses
hostname -I

# show process uptime
ps -o lstart ${PID}

# debug network
while true; do
  ip a && iptables -S | tee "/tmp/iptables_$(date -Is)"
  sleep 1
done

# load check
dstat -cdn -D sda -N eno1 -C total --top-cpu --top-io --top-mem -f 10
nethogs
nload eno1 -o 10000 -t 1000

# show my extenal / outgoing IP
curl https://ipinfo.io/ip

# get internal kernel device name for all block devices
lsblk -np --output KNAME | tac
lsblk -np --output KNAME -J

# edit sudoers file
visudo -f /etc/sudoers.d/nagios

# copy disc with dd
nice -n 19 ionice -c 3 dd if=/dev/sdc of=/dev/sdd bs=16384 status=progress

# get realpath of symlink
readlink -f /path/to/link

# change vfat partition label
mlabel -i /dev/sdd1 ::usb32gb

# get boot parameter
cat /proc/cmdline | sed -e 's/^.*BOOTIF=//' -e 's/ .*$//'

# Manuall compress big files
find /srv/log/*/202*/0{1..3}/ -type f -size +128M | xargs gzip

# gzip to specific directory / file
gzip -c file_in > /paht/to/file_out.gz

# get real interface mac address
ethtool -P eth0

# mount by label / partlabel
mount PARTLABEL=esp /mnt
mount LABEL=esp /mnt

# set command timeout
timeout 5 ping google.com

# output 10 chars
cat /tmp/input.txt  | cut -c -10

# show disk statistics
iostat -x /dev/nvme*

# test UDP port
nc -z -v -u  

# unlock LUCS disk
sudo udisksctl unlock -b /dev/sdb1
sudo mount /dev/dm-0 /mnt/

# lock LUCS disk
sudo udisksctl unmount -b /mnt
sudo udisksctl lock -b /dev/sdb1

# LUCS v2
sudo apt-get install -y cryptsetup

# decrypt volume
sudo cryptsetup luksOpen /dev/sdb1 luks1
sudo mount /dev/mapper/luks1 /mnt

# lock volume
sudo umount /mnt
sudo cryptsetup luksClose luks1

# test port 80 (webserver with nc)
apt install -y netcat-traditional
while true ; do nc -l -p 80 -c 'echo -e "HTTP/1.1 200 OK\n\n $(date)"'; done

# ftp
cat ~/.netrc 
machine 192.168.0.10
login ftp_user_1
password xxxxxxx

ftp 192.168.0.10 <

<strong>Ad-hoc HTTP server</strong>
https://github.com/weihanglo/sfz
<code>
URL=https://github.com/weihanglo/sfz/releases/download/v0.3.0/sfz-v0.3.0-x86_64-unknown-linux-gnu.tar.gz

wget ${URL} -qO- | tar -xz -C /tmp
/tmp/sfz -a -L -b 0.0.0.0 -p 8080

# Create HTTP server with python
python -m http.server  --directory

Tmux

# split window horizontaly
ctrl+b
shift+"

# split window verticaly
ctrl+b
shift+%

# multiplex commands
ctrl+b
shift+:
set synchronize-panes
#set synchronize-panes off

# Show double line in file
sort  | uniq -c

Links
https://how.wtf/articles.html