Linux Short Reference

1 FILE AND DATA HANDLING

1.1 find hints

find suid bits

find / -xdev -perm -4000 -exec ls -l {} \;

find world writeable files

find / -xdev -perm -o+w -and -not \( -type l -or -type s -or -perm -o+t \) -exec ls -ld {} \;

Find Duplicate Files (based on size first, then MD5 hash)

find -not -empty -type f -printf "%s\n" | sort -rn | uniq -d | xargs -I{} -n1 find -type f -size {}c -print0 | xargs -0 md5sum | sort | uniq -w32 --all-repeated=separate

remove files older than 60 days

find /var/log/ -type f -name '*.log' -ctime +60 -exec rm -f {} \;

show what have been modified last 60 minutes

find / -mmin +60 -type f

find files with lines longer than

find . -type f -exec grep -l '.\{80\}' {} \;

find core dumps

/bin/nice -19  /usr/bin/find / -type f -print 2>/dev/null | egrep  -r '/core\.[0-9]{2,}' | /usr/bin/xargs ls -l

rename files with spezial characters in it

convmv --notest -f latin1 -t utf8 *.pdf

remove umlauts from file/folders

find . -type d | while read dir; do rename 's/ö/oe/g;s/Ö/Oe/g;s/ü/ue/g;s/Ü/Ue/g;s/ä/ae/g;s/Ä/Ae/g' "$dir"; done
find . -type f | while read file; do rename 's/ö/oe/g;s/Ö/Oe/g;s/ü/ue/g;s/Ü/Ue/g;s/ä/ae/g;s/Ä/Ae/g' "$file"; done

1.2 vim hints

Save a file you edited in vim without the needed permissions

:w !sudo tee %

1.2.1 encrypt with vim

The safest way to do this is to add the following to your ~/.vimrc file:

set cm=blowfish2
set viminfo=
set nobackup
set nowritebackup

Now you can crypt file with vim -x filename

1.2.2 show whitespaces

:set listchars=eol:¬,tab:>·,trail:~,extends:>,precedes:<,space:␣
:set list

1.2.3 ansible config

The safest way to do this is to add the following to your ~/.vimrc file:

autocmd Filetype yml setlocal ai sw=2 et

1.3 trash a open logfile

cat /dev/null > logfile
echo -n > logfile
> logfile #bash

1.4 rhel8 auditd logrotate generations

auditd is keeping logs, which may fill up the os partitions

sed -i 's/max_log_file_action =.*/max_log_file_action = rotate/' /etc/audit/auditd.conf
pkill -9 -f /sbin/auditd
ps -ef | grep -i auditd
systemctl start auditd.service
systemctl status auditd.service
ls -la /var/log/audit/

1.5 Make 'less' behave like 'tail -f'

less +F somelogfile

1.6 count processes per user

ps hax -o user | sort | uniq -c

1.7 Get the 10 biggest files/folders for the current direcotry

du -sm * .[^\.]* | sort -n | tail

2 DATA MANIPULATION AND PARSING

2.1 remove commented lines from file

sed 's/#.*$//' -e '/^$/d' -e '/^\s*#.*$/d' /etc/file.cfg

2.2 unix2dos with sed

sed -i 's/$/\r/' file.txt

2.3 dos2unix with sed

sed -i 's/\r//' file.txt

2.4 search and replace onliner

perl -pi -w -e 's/search/replace/g;' *.txt

or

sed -i 's/search/replace/g;' *.txt

2.5 replace multiline pattern

perl -i -pe 'BEGIN{undef $/;} s/START_PATTERN.*END_PATTERN/REPLACE_STRING/smg' file1.txt

2.6 Display a block of text with AWK

awk '/start_pattern/,/stop_pattern/' file.txt

2.7 Delete Block of Text with sed

cat MYFILE |sed '/START_PATTERN/,/END_PATTERN/d'

2.8 Prettify an XML file

tidy -xml -i -m [file]
xmllint --format [file]

2.9 Prettify an JSON file

cat file.json | python -m json.tool

2.10 show changelog from pending updates

echo n | yum update --changelog | sed '1,/Changes in packages about to be updated:/d' | sed '/Running transaction check/,$d'

2.11 yum snippets

• list installed packages and repo

repoquery -a --installed --qf "%{ui_from_repo} %{name}"
yum list installed | egrep -i 'epel|ovirt'

2.12 find differences between two files

classical side to side comparsion

diff -y file1 file2

enhanced comparsion with highlighting (package: vim-enhanced)

vimdiff file1 file2
gvimdiff file1 file2

Compare a remote file with a local file

ssh user@host cat /path/to/remotefile | diff -y /path/to/localfile -

2.13 convert txt to pdf

cal > cal.txt
enscript -o cal.ps cal.txt 
ps2pdf cal.ps 

2.14 remove color from bash output (escape sequences)

color-script.sh  | col -b | sed 's/0;[0-9]*m//g'

2.14.1 convert bash color output into html file

yes | ansible-playbook csv-runner-baseEvpn.yml  | tee >(aha > ansible_example_output.html)

2.15 Quick access to the ascii table

man ascii

2.16 get network interface ip

/sbin/ifconfig $DEVICE | awk '/inet/ { print $2 } ' | sed -e s/addr://

3 BACKUP, HARD DISK AND SAN DISK HANDLING

3.1 backup helper script

echo '#!/bin/sh
cp -av "$1" "$1.$(date +%Y%m%H%M%S)"
' > /usr/local/bin/backup
chmod 755 /usr/local/bin/backup

3.2 create sparsed image files with dd

dd if=/dev/zero of=xen-guest.img bs=1 count=0 seek=8G

3.3 create random data fast

# create 50Gig of data with 5 threads
for i in {1..5} ; do ( openssl enc -aes-256-ctr -pass pass:"$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64)" -nosalt < /dev/zero 2>&1 | dd of=/tmp/file_10G.$i bs=1M count=10k iflag=fullblock ) & done

# create 5 data generating threads which create infinite data files
for i in {1..5} ; do ( openssl enc -aes-256-ctr -pass pass:"$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64)" -nosalt < /dev/zero 2>&1 | dd of=/tmp/file.$i) & done
# stop data generation, started with comand above
pkill -9 -f dd

3.4 ultra fast file copy of large file volumes

tar --ignore-failed-read -C $SRC_DATA/ -cf - . | mbuffer -L -s 256k -m 1G -P 85 | tar --ignore-failed-read -C $DST_DATA/ -xf -

3.5 send status mail with block device and multipath information

(echo --detect_multipath--;detect_multipath;echo --df-h--;df -h;echo --mount--;mount; \
echo --fstab--;cat /etc/fstab;echo --mdstat--;cat /proc/mdstat;echo --partitions--; \
cat /proc/partitions)| mail -s `uname -n` chris.ruettimann-at-poweroff.com

3.6 bonnie++ example

bonnie++ -uroot -x5 -m hihhlx05 -n 100:400k:200k:1024 -d /aim/data01/test
bonnie++ -d /tmp -s 4G -n 0 -m TEST -f -b -u root
bonnie++ -d /srv  -s 4G -n 0 -m $(uname -n) -f -b -u root
echo "[RESULTS]" | bon_csv2html > [OUTPUT]

3.7 read HD smart status

smartctl -a /dev/sda
smartctl -H /dev/sda

3.8 read udev disk attributes

read disk serial

udevadm info --query=all --name=/dev/sda | grep ID_SERIAL_SHORT | cut -d= -f2

3.9 backup acl with star

star -Hexustar -acl -c f=home.star /home/
star -acl  -c f=home.star

3.10 save and restore acl attributes

# be carefull, restore deleted my suid permissions :)
getfacl -R . >acl.txt
setfacl --restore acl.txt
getfacl -R $(ls -d /* | egrep -v 'dev|proc|selinux|sys|lost+') > /etc/acl.txt

3.11 turn off auto hard disc boot scanning for ext3 and reduce root preserved space

tune2fs -c 0 -i 0 -m 0 /dev/VG0/data

3.12 set dir_index on ext3 for faster directory listing

tune2fs -O dir_index /dev/VG0/root
fsck.ext3 /dev/VG0/root

3.13 show extended superblock information of partition

debugfs -R stats /dev/VG0/root

3.14 modify root preserved space on ext3 partition

tune2fs -m 6 /dev/sda1
tune2fs -l /dev/sda3 | grep -i block

3.15 mark bad blocks on degrading hard disk

umount /dev/sda1
e2fsck -cc /dev/sda1

3.16 mirror website with wget

 wget --random-wait -r -U Mozilla -e robots=off --span-hosts --domains miyuru.lk --convert-links https://www.miyuru.lk/geoiplegacy/

4 NETWORKING NOTES

4.1 WAN SpeedTest Adventures

yum -y install epel-release
yum -y install python2-speedtest-cli python-setuptools
speedtest-cli --list 2>/dev/null | egrep -i 'switzerland|Germany|Italy|France' | cut -d')' -f1 | sort -un | while read id
do 
  echo "------ $( speedtest-cli --list 2>/dev/null | egrep "^$id\) | $id\)" ) $(date) ------" 
  speedtest-cli --server $id --simple 2>/dev/null
done

4.2 Private Key handling with keychain in .bashrc

• SSH Keys

keychain -Q -q ~/.ssh/id_dsa < /dev/null
[ -f $HOME/.keychain/$HOSTNAME-sh ] && source $HOME/.keychain/$HOSTNAME-sh

• GPG Keys

#gpg --list-keys
keychain --agents gpg 297E196D
[ -f $HOME/.keychain/$(uname -n)-sh-gpg ] && source $HOME/.keychain/$(uname -n)-sh-gpg

4.3 Use keychain to protect your Ansible SSH private Keys

• on control node, install keychain

yum install keychain

• on control node, with the ansible user do add keychain

vi .bashrc
------
keychain -Q -q ~/.ssh/id_rsa < /dev/null
[ -f $HOME/.keychain/$HOSTNAME-sh ] && source $HOME/.keychain/$HOSTNAME-sh
------

• protect your ssh key

[ansible-adm@control-node ~]$ ssh-keygen -p -f .ssh/id_rsa
Key has comment '.ssh/id_rsa'
Enter new passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved with the new passphrase.

• log out and log in, it will ask for pass phrase

[ansible-adm@control-node ~]$ exit
Connection to control-node closed.
[user@jump ~]$ ansible-adm@control-node 
Last login: Thu Sep 29 11:39:00 2016 from 17.2.25.25
Enter passphrase for /home/ansible-adm/.ssh/id_rsa: 

• now you can add cronjobs that run ansible commands

remember you have to login once after every reboot of control-node

crontab -e
------
SHELL="/bin/bash"
PATH=/usr/local/bin:/bin:/usr/bin:$HOME/bin
* * * * * . $HOME/.bashrc; cd dep-adhoc ; ansible all -m ping >> cron.out 2>&1
------

• check results

cat cron.out 

server1.example.com | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
localhost | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}

4.4 Show current Details of all NICs (COS7)

( echo MAC: MEDIA: STATUS: SPEED: NIC: 
  ip link show | egrep '^[0-9]+:' | awk '{print $2}' | cut -d: -f1 | grep -v lo | while read if
  do
    MAC=$(ip addr show $if | grep link/ether | awk '{print $2}' )
    MSPEED=$(ethtool $if | grep baseT | cut -d: -f2- | tail -n1 | awk '{print $1}')
    TYPE=$( ethtool $if | grep 'Supported ports:'  | awk '{print $4}')
    LINK=$(ethtool $if | grep -q 'detected: yes' && echo UP || echo DOWN)
    echo "$MAC $TYPE $LINK $MSPEED $if" 
  done ) | column -t

4.5 Generate random MAC address

printf 'DE:AD:BE:EF:%02X:%02X\n' $((RANDOM%256)) $((RANDOM%256))
date +%s |md5sum|sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'

4.6 Serve current directory tree at by http port 8000 on local machine

python3 -m http.server
python2 -m SimpleHTTPServer

4.7 block known bad hosts from reaching your machine

wget -qO - http://infiltrated.net/blacklisted | awk '!/#|[a-z]/&&/./{print "iptables -A INPUT -s "$1" -j DROP"}'

4.8 find routing decision

ip route show match 1.2.3.4
ip route get 1.2.3.4

4.9 set ip address on the fly

ifconfig eth0:1 192.168.111.1 netmask 255.255.255.0 up
route add -host 192.168.33.1 dev eth0:1
route add -net 192.168.111.0 dev eth0:1
route add default gw 192.168.0.254

4.10 pipe tar via ssh

put data

cd /usr/local/stuff
tar cfz - . | ssh -c blowfish remote "cd /backup && tar xfz -"

get data

ssh -c blowfish remote "cd /usr/local/stuff && tar cf - ." | tar xfz -

4.11 pipe disc image via ssh

get data

ssh  root@get-disk-from dd bs=8192 if=/dev/sda | dd bs=8192 of=/dev/vg01/backup_sda

put data

dd bs=8192 if=/dev/sda | ssh  root@put-disc-to dd bs=8192 of=/dev/vg01/backup_sda
dd if=/dev/sda1 | buffer -s 64k -S 10m | ssh root@put-disk-to "cat > /dev/image"
dd if=/dev/sda1 bs=4M | lzop -c | ssh root@put-disk-to "lzop -dc | dd of=/dev/sda1 bs=4M"

4.12 escape remote console

4.12.1 by telnet / Xen VM Console

Escape character is '^]'.
and that means: <CTRL>+<AltGr>+]

4.12.2 access ilo by ssh

ssh user@ilo-host

</>hpiLO-> remcons
Starting remote console
Press 'ESC (' to return to the CLI Session

4.12.3 from cyclades

~.

4.13 install public key on remote machine

ssh-copy-id username@hostname

4.14 Resume rsync of a big file

rsync --partial --progress --rsh=ssh  $file_source $user@$host:$destination_file

4.15 Useful Rsync Example

rsync -xXvrltH --exclude="lost+found" --delete -pgo --stats -D --numeric-ids $SRC_DIR/ $DST_DIR

4.16 install ssh pub keys from users github account

curl -s https://github.com/joe-speedboat.keys | while read key ; do 
   grep -q "$key" ~/.ssh/authorized_keys && echo "Key did exist: $key"|| (echo "$key" >> ~/.ssh/authorized_keys ; echo "Key added: $key" )
done

4.17 port forwarding

ssh reverse tunnel

#build the ssh reverse tunnel
user@destination$ ssh -R 2222:localhost:22 middleuser@middle
user@destination$ ssh -N -f -R 2222:localhost:22 middleuser@middle
#connect to the ssh reverse tunnel and jump to destination
middleuser@middle$ ssh destinationuser@localhost -p2222
user@notebook$ ssh destinationuser@middle -p 2222

ssh port forwarding

ssh -g -L 80:127.0.0.1:3128 jump@zen.bitbull.ch -p23
ssh -g -L local_port:remote_host:remote_port  user@dst_host -p23
  -g  >  Allows remote hosts to connect to local forwarded ports
  -L  >  [bind_address:]port:host:hostport
  -p  >  use different port for ssh connection

rinetd

simple port forwarder in userspace for linux and windows
homepage: http://www.boutell.com/rinetd/

stunnel

traffic redirection via ssl tunnel
homepage: http://www.stunnel.org/
zB: MUA -> localhost:110 -> stunnel( localhost:110 -> destination:995)

proxytunnel

building tunnels over http proxies for ssh, pop3 ...
homepage: http://proxytunnel.sourceforge.net/

4.18 remember ssh private key passphrase on console

#start the agent
set | grep SSH_ || eval $(ssh-agent)
#now, give the pass passphrase
ssh-add
# done
# or do the alias way eq: vi /etc/bashrc
alias skey='ssh-agent > /tmp/.k ; . /tmp/.k ; rm -f /tmp/.k ; ssh-add'

4.19 remember ssh private key passphrase in gnome session

# install rpm
yum -y install openssh-askpass
# now add asking dialog to gnome user
# Gnome Menu > System > Preferences > More Preferences > Sessions > Tab:Startub Programs > Add
# after restart of gnome: Gnome Menu > System > Preferences > More Preferences > Sessions > Tab:Current Session (ssh-add) -> set order to 90
# type in: /usr/bin/ssh-add

4.20 validate date of ssl certificate

echo | openssl s_client -connect www.google.com:443 2>/dev/null |openssl x509 -dates -noout

ssl-test() { curl -kvv --max-time 2 https://$1 2>&1 | egrep 'issuer:|expire date:|start date:|subject:' ;}

ssl-test www.google.ch
   *  subject: C=US; ST=California; L=Mountain View; O=Google LLC; CN=*.google.ch
   *  start date: Jan  5 12:14:12 2021 GMT
   *  expire date: Mar 30 12:14:11 2021 GMT
   *  issuer: C=US; O=Google Trust Services; CN=GTS CA 1O1

4.21 SSL/TLS debugging

• Get all LISTEN Ports and test for Certificate details by web request

IF=$(ip r | grep default | awk '{print $5}')
IP=$(ip addr show $IF | grep "inet " | awk '{print $2}' | cut -d'/' -f1)
lsof -i -P -n 2>/dev/null | grep LISTEN | grep -v 127.0.0.1 | awk '{print $9}' | cut -d: -f2 | sort -u | while read port 
do 
   echo "---------- https://$IP:$port "
   lsof -i -P -n 2>/dev/null | grep LISTEN | grep :$port 
   curl -m3 -k -vv https://$IP:$port 2>&1 | grep -A4 'Server certificate:'
done

4.22 tcpdump

# max. 100MB /  full length and host x.x.x.x   
tcpdump -C 100 -w dump.dat -i eth0 -s 0 -XX udp port 514
tcpdump host client.example.com and port 80
tcpdump ip host bevo and bevo master
tcpdump ip and not net 127.0.0.0

To print the start and end packets (the SYN and FIN packets) of each TCP conversation that involves a non-local host

tcpdump \(tcp[13] \& 3 !=0 and not src and dst net localnet\)

To print all ICMP packets that are not echo requests or replies (not ping packets)

tcpdump \(icmp[0] !=8 and icmp[0] !=0\)

Dump Traffic for later view in Wireshark

tcpdump -p -s0 -w tcpdump.cap

4.23 conunt network connections for each host

netstat -an | grep ESTABLISHED | awk '{print $5}' | awk -F: '{print $1}' | sort  | uniq -c | sort

4.24 List the number and type of active network connections

netstat -ant | awk '{print $NF}' | grep -v '[a-z]' | sort | uniq -c

4.25 nmap network discovery

• find web servers on subnet

nmap -sS -T5 -P0 -p80 -oG - 192.168.1.1-254 | grep open

4.26 monitor network activity of applications

while true 
do 
   date
   diff  <(lsof -i) <(sleep 5; lsof -i) 
done

4.27 Analyse an Apache access log for the most common IP addresses

tail -10000 access_log | awk '{print $1}' | sort | uniq -c | sort -n | tail

4.28 show apps that are using network connection

lsof -P -i -n

4.29 show your wanip

curl ifconfig.me

4.30 scan open outgoing ports

for i in {1..1024}; do wget -qO- -T0.5 -t1 portquiz.net:$i >/dev/null 2>&1 && echo $i open ; done

4.31 curl webdav examples

• Reading Files/Folders on Webdav Server

curl 'https://example.com/webdav'

• Deleting Files/Folders on Webdav Server

curl -X DELETE 'https://example.com/webdav/test'

• Similarly for deleting file test.txt

curl -X DELETE 'https://example.com/webdav/test.txt'

• Renaming File on Webdav Server

curl -X MOVE --header 'Destination:http://example.org/new.txt' 'https://example.com/old.txt'

• Creating new foder on Webdav Server:

curl -X MKCOL 'https://example.com/new_folder'

• Uploading File on Webdav Server

curl -T '/path/to/local/file.txt' 'https://example.com/test/'

• CURL --Options

• Username/Password

curl --user 'user:pass' 'https://example.com'

• HTTP Authentication

curl --user 'user:pass' 'https://example.com' --basic
curl --user 'user:pass' 'https://example.com' --digest

• curl decide the authentication

curl --user 'user:pass' 'https://example.com' --anyauth

• Get Response Code

curl --user 'user:pass' -X DELETE 'https://example.com/test' -sw '%{http_code}'

4.32 nextcloud upload to shared link with curl

• acess with no password -> copy uuid url:

https://cloud.domain.org/s/EaaddddcMMt2aZb

• Upload file with this curl oneliner (adjust token and url):

curl -u EaaddddcMMt2aZb: -H "X-Requested-With: XMLHttpRequest" "https://cloud.domain.org/public.php/webdav/" -T mynotes.txt

• Get friday beer, beacause your work is save now !!!

(reuse this instruction every friday :-)

4.33 use netcat port check

nc -vvn -z 10.202.3.40 80 #old syntax
nc -w3 -i3 --recv-only $DSL_IP $DSL_PORT 2>/dev/null | grep -q Login: #new syntax

4.34 use netcat and dd to check network performance

on dst:

nc -l -vv -p 8080 >/dev/null

on src:

dd if=/dev/zero bs=1M count=10240 | nc  dst-host 8080

4.35 execute comands on windows server

# homepage: http://sourceforge.net/p/winexe/wiki/Home/
# download: http://download.opensuse.org/repositories/home:/ahajda:/winexe/
# download: http://repo.openpcf.org/repository/ext/openpcf/
echo -e 'WDSUtil /Add-Device /Device:w-nb-05 /ID:74867a2a18a0 /OU:"OU=Computer,DC=domain,DC=local" \n exit' | winexe -U DOMAIN/Administrator%SuperDuper123 //10.0.0.41 cmd

5 SYSTEM NOTES

5.1 Get Hyperviser Information from within VM

cat /sys/class/dmi/id/sys_vendor

5.2 install 32bit libraries on CentOS7

yum install redhat-lsb.i686

5.3 python ssl broken, how to repair

pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org <package_name>

5.4 trash rsyslog msg pattern

• /etc/rsyslog.d/mydomain.conf

#auth,authpriv.* @syslog.mydomain.ch:1516

:msg, contains, "pam_unix(cron:session)" ~
:msg, contains, ": uid: missing" ~
*.* @syslog.mydomain.ch:1516

5.5 Show File System Hierarchy

man hier

5.6 Execute a command without saving it in the history

<space>command

5.7 Show Date in bash history

echo 'export HISTTIMEFORMAT="%d/%m/%y %T "' >> ~/.bash_profile
. ~/.bash_profile
history

5.8 execute command at given time

echo "ls -l" | at 22:00 30.12.13

5.9 insert newline into cronjob

30 1 * * *  /usr/bin/ssh admin@10.0.0.1 'execute reboot ^My' >/dev/null 2>&1
# CTRL-v, CTRL-m

5.10 Salvage a borked terminal

reset

5.11 Rapidly invoke an editor to write a long command

EDITOR=vim
ctrl-x e

hold ctrl, then hit x, then hit e, release ctrl

5.12 run last command as root

 sudo !!

forgot to login as root, take it easy

5.13 change into shell of a disabled system user

[root@cloud ~]# getent passwd nginx
nginx:x:995:993:Nginx web server:/var/lib/nginx:/sbin/nologin
[root@cloud ~]# su - nginx
This account is currently not available.
[root@cloud ~]# su -s /bin/bash nginx
bash-4.2$ id
uid=995(nginx) gid=993(nginx) groups=993(nginx) context=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023

5.14 Runs previous command but replacing

^foo^bar
 # example(replace once):
vi 7etc/hosts
^7^/

!!:gs/foo/bar
 # example(replace many):
vi 7etc7hosts
!!:gs#7#/

5.15 remember directroy for later jump back

cd /very/complex/path/
pushd . #remember it
cd /tmp/wtf/
pushd . #remember it
cd /etc/
popd #jump to /tmp/wtf/
popd #jump to /very/complex/path/

5.16 configure date and time

get system time from ntp-server

ntpdate pool.ntp.org

manually set system time

date MMDDhhmmYYYY
date MMDDhhmm
# example 27.12.1975 08:00
date 122708001975

write system time into bios (hw clock)

hwclock -w

set timezone

cd /etc
ln -sf /usr/share/zoneinfo/EST localtime

5.17 show disk usage in nice format

df -hP | column -t

5.18 find system installation date

tune2fs -l $(df -P / | tail -n1 | cut -d' ' -f1 ) | grep 'Filesystem created:'
ls -lact --full-time /etc | tail -1 | awk '{print $6,$7}'
rpm -qi basesystem | grep Install

5.19 create nice overview of system hardware

lshw -html > hardware.html

5.20 Processor / memory bandwidthd? in GB/s

dd if=/dev/zero of=/dev/null bs=1M count=32768

5.21 ultimative system monitoring comand

dstat -f -M time,cpu,net,disk,sys,swap,page,load,proc,topcpu --output $(date '+%Y.%m.%d-%H.%M')-dstat.csv
dstat -cdngymsp --lock --tcp --output $(date '+%Y.%m.%d-%H.%M')-dstat.csv

5.22 strace examples

strace -ff -e trace=write -e write=1,2 -p SOME_PID
strace -e open ls
strace -e trace=open,read ls /home
strace -o output.txt ls 
strace -f -p 1725 -o firefox_trace.txt # f: follow process
strace -t -e open ls /home #timestamp
strace -c ls /home # statistics
strace -f -t -e trace=file systemctl restart SuSEfirewall2 2>&1 | grep open | cut -d'"' -f2

5.23 useful alias to view system processes

alias px='ps -eo ruser,pid,rss,vsz,pcpu,tty,args | grep -v grep | grep -e COMMAND -e'

5.24 show bios and hardware information

dmidecode

5.25 Show Free Memory in Percent

MEM=($(free -t| grep ^Mem)) ;echo FreeMemPct: $(((${MEM[1]} - ${MEM[2]}) * 100 / ${MEM[1]} ))
free -t | awk '/Mem/{print ($2-$3) * 100.0 / $2}'

5.26 Free Up Cache Memory

To free pagecache:

echo 1 > /proc/sys/vm/drop_caches

To free dentries and inodes:

echo 2 > /proc/sys/vm/drop_caches

To free pagecache, dentries and inodes:

echo 3 > /proc/sys/vm/drop_caches

5.27 Manually create Out of memory event (OOM)

swapoff -a
# 1.1 is the amount of memory to use
stress --vm-bytes $(awk '/MemAvailable/{printf "%d\n", $2 * 1.1;}' < /proc/meminfo)k --vm-keep -m 1

5.28 search memory for given readable strings

dd if=/dev/mem | cat | strings | grep xyz

5.29 Mounting a disk image containing several partitions

#list partition table of image file
fdisk -l disk.img
kpartx -l disk.img

#create devices
kpartx -a -v disk.img
ls -all /dev/mapper/

#do what you need to do
mount /dev/mapper/loopXXX /mnt/tmp -o loop
sleep 600
umount /mnt/tmp

#delete devices when work is done
kpartx -d -v disk.img
ls -l /dev/mapper/

5.30 Mount a temporary ram partition

mount -t tmpfs tmpfs /mnt -o size=1024m

5.31 cat top output into a text file

top -b -n1 > /tmp/top.txt

5.32 sudo and ulimit

sudo bash -c 'ulimit -n 8192; sudo -u username ./startup-script'

now its possible to use it with /etc/sudoers :)

5.33 disable requiretty on sudo for one user/group

this is the cronjob which has to be run as monitoring user

crontab -e -u monitoring
---
*/15 * * * * /usr/bin/sudo /bin/su -l oracle -c "/usr/local/mon/bin/oracle-check.sh oratbs DB01 APP" > /usr/local/mon/data/oracle-check.oratbs.DB01.APP.tmp
---

this are the sudo settings you need

visudo
---
Defaults:mon   !requiretty
mon ALL=(ALL) NOPASSWD:/bin/su -l oracle -c /usr/local/mon/bin/oracle-check.sh DB01 APP
--

5.34 sudo: allow single comand with arg

 mon ALL=(ALL) NOPASSWD:/usr/bin/sudo /usr/bin/nmap -O

5.35 detach running process so that you can logout

sleep 600
# <CTRL>+<Z>
[1]+  Stopped                 sleep 600
# bg
[1]+ sleep 600 &
# jobs
[1]+  Running                 sleep 600 &
# disown -r
# jobs
ps -ef | grep sleep
root     29081 28991  0 13:23 pts/0    00:00:00 sleep 600

5.36 avoid bash auto logout

5.36.1 TMOUT variable

• Based on RHEL8
• Logout message: timed out waiting for input: auto-logout

echo $TMOUT
man bash
# rpm -qf /etc/profile.d/tmout.sh
  file /etc/profile.d/tmout.sh is not owned by any package
# cat /etc/profile.d/tmout.sh
  # Set TMOUT to 900 per security requirements
  TMOUT=900

5.36.2 SSH Config

[root@svtuxjump01p ~]# grep -B1 Alive /etc/ssh/ssh*_config
/etc/ssh/ssh_config:Host *
/etc/ssh/ssh_config:   ServerAliveInterval 60
--
/etc/ssh/sshd_config:ClientAliveInterval 60

fgrep -r Alive /etc/ssh/

5.37 give important system message to tty and switch display

clear >/dev/tty9
echo '
      ***************************************************
      *        SYSTEM WILL SHUT DOWN AT 19.30           *
      *                                                 *
      *        SAVE YOUR WORK AND EXIT NOW !!!          *
      ***************************************************' >/dev/tty9
chvt 9
sleep 5
chvt 7

5.38 bash prompt examples

for documentation

export PS1='#\u;\h;\D{%Y.%m.%d %H:%M.%S};\w\n '

to mark important hosts

export PS1='\e[0;45m \e[0;30m[\u@\h \w]\$ '

or

export PS1="\[$(tput setaf 1)\]$PS1\[$(tput sgr0)\]"

5.39 log current shell session into file

script -a -f $HOME/console.log

5.40 find procs having too many open files

lsof +c15 > lsof.tmp
for i in $(cat lsof.tmp | cut -d' ' -f1 | sort -u ); do echo $i : $(cat lsof.tmp | grep "^$i " | wc -l); done | sort -n -t: -k2 | column -t | tail
rm -f lsof.tmp

5.41 get memory usage for many procs

ps -ylC php-fpm --sort:rss | awk '{sum+=$8; ++n} END {print "Tot="sum"("n")";print "Avg="sum"/"n"="sum/n/1024"MB"}'

5.42 generate passwords

 genpasswd() {
	local l=$1
       	[ "$l" == "" ] && l=20
      	tr -dc A-Za-z0-9_ < /dev/urandom | head -c ${l} | xargs
 }

tr -dc 'A-NP-Za-np-z1-9.,;:+/()=?[]{}_-' < /dev/urandom | head -c16 | xargs

openssl rand -base64 24

5.43 selinux handling

setenforce 0 # set permissive
semanage fcontext -l | grep /var/www
semanage fcontext -d -t httpd_sys_content_t "/data"
semanage fcontext -a -t httpd_sys_content_t "/data"
semanage fcontext -a -t httpd_sys_content_t "/data/www/([^/]*/)?www(/.*)?"
semanage fcontext -a -t httpd_config_t "/data/www/([^/]*/)?conf(/.*)?"
semanage fcontext -a -t httpd_log_t "/data/www/([^/]*/)?logs(/.*)?"
restorecon -Fr /data
ls -lZ /data
for i in $(rpm -ql policycoreutils | grep bin/ ); do man -k $(basename $i); done
semanage user -l httpd
systemctl restart httpd
sealert -a /var/log/secure
# fix if error
setenforce 1
sed -i 's/^SELINUX=.*/SELINUX=Enforcing/' /etc/selinux/config
grep ^SELINUX= /etc/selinux/config
getenforce

5.44 selinux altertin

yum -y install setroubleshoot-server setroubleshoot-plugins setroubleshoot-doc

vi /etc/setroubleshoot/setroubleshoot.conf

[email]
recipients_filepath = /var/lib/setroubleshoot/email_alert_recipients
smtp_port = 25
smtp_host = mail.domain.local
from_address = selinux@domain.local
subject = [DOMAIN] SELinux AVC Alert

vi /var/lib/setroubleshoot/email_alert_recipients
------
support@ict4u.li
------

service messagebus restart

5.45 bash notes

some nice comands

[ $# -lt 1 ] || cat $FILE
egrep -qi 'error|fail' $LOG && touch /etc/clean

dont log duplicate entrys in .bash_histroy

echo 'HISTCONTROL=ignoreboth' >> $HOME/.bashrc

set variables in variables

$> X=horse
$> eval $X=23
$> echo $X : ${!X}
horse : 23

variables in functions

var=hello 
foo () { echo "${!1}"; } 
foo var 
   hello

work with arrays

HOSTS=($(egrep -v '^#|^$' /etc/hosts | awk '{print $2}'))
for HOST in ${HOSTS[*]} ; do
   NR=$(($NR + 1))
   echo "   $NR)   $HOST"
done

echo -n "choose a host: " ; read DEST
echo "ssh ${HOSTS[$(($DEST -1))]}"

5.46 Perform a branching conditional

true && { echo success;} || { echo failed; }

5.47 password handling

lock the account

usermod -L <username>

Change the password expiration date to 0 to ensure the user changes the password during the next login

chage -d 0 <username>

unlock the account

usermod -U <username>

set password from within a script

echo my-secret-password | passwd --stdin <username>

5.48 Backup/Restore Packages of Debian System

Nice comand to colone and reset debian based systems

5.48.1 Backup

dpkg --get-selections > /etc/dpkg-list.txt

5.48.2 Restore

/usr/bin/dpkg --clear-selections
/usr/bin/dpkg --set-selections < /etc/dpkg-list.txt
/usr/bin/dpkg --get-selections | sed -e 's/deinstall/purge/' > /tmp/dpkg-list.txt
/usr/bin/dpkg --set-selections < /tmp/dpkg-list.txt
rm -f /etc/dpkg-list.txt
/usr/bin/apt-get dselect-upgrade

6 DIVERSE NOTES

6.1 lock file handling with noclober and trap

if ( set -o noclobber; echo "$$" > "$lockfile") 2> /dev/null; 
then
   trap 'rm -f "$lockfile"; exit $?' INT TERM EXIT 

   critical-section
  
   rm -f "$lockfile"
   trap - INT TERM EXIT
else
   echo "Failed to acquire lockfile: $lockfile." 
   echo "Held by $(cat $lockfile)"
fi

6.2 prevent bash script from running twice

# this has to be placed on top of script
LCK_FILE=/var/run/$(basename $0).run
test -f $LCK_FILE
if [ $? -eq 0 ] # if lockfile is present, check if valid
then
   ps $(cat $LCK_FILE)
   if [ $? -ne 0 ] # check if PID of lockfile exists
   then
      logger -t $(basename $0) "WARNING: lockfile has invalid pid PID=$(cat $LCK_FILE), I delete lockfile and run the script"
      rm -f $LCK_FILE
   else
      logger -t $(basename $0) "INFO: script is already running, I will exit the script now" 
      exit 1
   fi
fi
trap 'rm -f "$LCK_FILE"; exit $?' INT TERM EXIT
echo $$ > $LCK_FILE

6.3 redirect script output within the script

#!/bin/bash
logfile=$$.log
exec > $logfile 2>&1
echo main script starts here

6.4 reduce PDF file size

gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4  -dNOPAUSE -dQUIET -dBATCH -sOutputFile=output.pdf input.pdf
gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/screen -dNOPAUSE -dQUIET -dBATCH -sOutputFile=output.pdf input.pdf
gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/ebook -dNOPAUSE -dQUIET -dBATCH -sOutputFile=output.pdf input.pdf
gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/printer -dNOPAUSE -dQUIET -dBATCH -sOutputFile=output.pdf input.pdf

6.5 change keyboard layout on the fly

# for console
loadkeys sg-latin1

#for X11
setxkbmap -layout ch

6.6 cli clipboard handling with x11

This is for use with pipes and stdout

alias copyc='xclip -sel clip'
alias pastec='xclip -o -sel clip'

Example:

cat /etc/hosts | copyc
pastec > myhosts

It works with mouse as well :-)

6.7 set terminal tab label within console to user@host

PROMPT_COMMAND='echo -ne "\033]0;$USER@$HOSTNAME\007"'

6.8 reassign pipe key from AltGr-1 to AltGr-7

xmodmap -e 'keycode 10 = 1 plus brokenbar exclamdown brokenbar exclamdown'
xmodmap -e 'keycode 16 = 7 slash bar seveneighths bar seveneighths'

6.9 change screen resolution to work with beamer

xrandr -s 1024x768 -r 60

or this:

# startup 2 head (Beamer + Laptop)
xrandr --output LVDS1 --mode 1024x768 --primary
# force use 1024x768 mode of the projector
xrandr --output VGA1 --mode 1024x768 --right-of LVDS1 || (xrandr --addmode VGA1 1024x768 && xrandr --output VGA1 --mode 1024x768 --right-of LVDS1)

#shutdown
xrandr --output VGA1 --off
xrandr --output LVDS1 --auto

6.10 share x11 session

this can be used to share linux desktop to network (viewonly)

x11vnc -passwd 53cret -rfbport 5900 -privremote -ncache -shared # view only
x11vnc -passwd 53cret -rfbport 5900 -ncache -shared # mouse and keyboard enabled

client can connect like this

vncviewer <xvnc-hostname>

6.11 Start KDE Session without User Password

ssh -X -Y -c blowfish root@<x11-host>
 apt-get install xserver-xephyr
 Xephyr -ac -screen 1024x768 -br  2> /dev/null :9999
 su - <username>
  DISPLAY=:9999 startkde4 

6.12 screen short cuts

C => CTRL
C+a "        Present  a list of all windows for selection.
C+a A        Allow the user to enter a name for the current window.
C+a 9        Switch to window number 0  -  9
C+a C-a      Toggle  to  the  window displayed previously.
C+a C-c      Create  a  new window with a shell and switch to that window.
C+a C-d      Detach screen from this terminal.
C+a h        Write a hardcopy of the current window to the file "hardcopy.n".
C+a H        Begins/ends logging of the current window  to the file "screenlog.n".
C+a k        Destroy current window.
C+a n        Switch to the next window.
C+a p        Switch to the previous window. 
C+a N        Show the number (and title)  of  the  current window.
C+a w        Show a list of window.
C+a ?        Show Screen key bindings
screen -S title   -> is starting a new screen with specific title

6.13 bash session recording

record the session

$> script -t 2> demo.timing -a demo.session
Script started, file is demo.session
$> echo do something
$> exit
Script done, file is demo.session

replay the session

$> scriptreplay demo.timing demo.session

6.14 mysql hints

repair and optimize mysql db

mysqlcheck -uroot -p@secret! -A -a -o -e -c -r --auto-repair

copy mysql db to other host in one ssh comand

mysqldump --add-drop-table --extended-insert --force --log-error=error.log -uUSER -pPASS OLD_DB_NAME | ssh -C user@newhost "mysql -uUSER -pPASS NEW_DB_NAME"

6.15 delay cron job by random minutes to spread load

1 12 * * * /bin/sleep ${RANDOM:0:2}m ; /usr/local/sbin/update.sh

6.16 Generate Montly Calendar from comandline

pcal -E -P a4 -B -F 1 -d /8 -t /18 -n /10 -a de -o cal.ps 2012

7 MULTIMEDIA NOTES

7.1 html snipesets

dir2html bash function

dir2html () {
   cd $1
   echo "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2//EN\">
         <HTML> <HEAD><TITLE>$(uname -n):$PWD</TITLE></HEAD><BODY>
         <H1><CENTER><FONT COLOR=blue><P><hr><B>$(uname -n):$PWD</B></P></H1>
         <hr><FONT COLOR=blue>" > index.html
   for LINK in $(ls -1 -d * | grep -v index | sort -r)
   do
      echo "<a href=\"$LINK\">$LINK</a><br>" >> index.html
   done
   echo "</font></center></BODY></HTML>" >> index.html
}

7.2 picture handling with imageMagick

resize all jpg pictures to exact 640x480 pix

convert -resize 640x480! *.jpg

resize all jpg pictures to 640 pix and keep ratio

convert -resize 640 *.jpg

cut select.gif with size=240x160 out of big.gif (Xstart=140 Ystart=100 )

convert big.gif -crop 240x160+140+100 +repage select.gif

convert gif to jpg

convert image.gif image.jpg

join 3 pics into 1 in one row

montage -mode concatenate -tile 1x3 top-pic.jpg mid-pic.jpg bottom-pic.jpg merged.jpg

label a image with custem text on bottom

convert noname.gif -size 15x15 xc:White -background White -append -draw "gravity South  text 0,0 'Calvin'" calvin.gif

create photo_index.html with thumbnails, pic-name and info from *.JPG

montage -label '%t\n%wx%h' -size 512x512 '*.JPG[120x90]' -geometry +5+5 -tile 5x  -frame 5  -shadow photo_index.html

write text into image

convert -draw "text 25,40 hello_world" -pointsize 36 source.png dest.png

7.3 capture current desktop as movie

ffmpeg -f x11grab -s wxga -r 25 -i :0.0 -sameq /tmp/out.mpg

7.4 movie converting

archive a crypted dvd (http://sourceforge.net/projects/dvdbackup)

dvdbackup -M

record iptv stream

avconv -i https://streams.iptv.com/get?channel=zdf_info -vcodec copy -ar 44100  -f mp4 movie.mp4

repair broken flv (osmc iptv simple recorder opts: -vcodec copy -acodec copy -strict experimental)

avconv -i broken.flv -codec copy -bsf:a aac_adtstoasc fix.mp4

rotate a movie

avconv -i in.avi -vf transpose=1 -vcodec ffv1 out.avi #other code:huffyuv

remove comercials from movie

avisplit -i DE\ -\ Mission\ Impossible\ II.avi -c -o out.avi -t 00:12:45-00:26:03,00:36:15-01:23:58,01:26:07-02:19:39

7.4.1 HandBrake

rip a DVD from dir

HandBrakeCLI -i ./Path/to/DVD/ -L -o My_DVD_Backup.avi

rip DVD into Xvid Avi

HandBrakeCLI -i /dev/dvd -e xvid -b 1024 -X800 -2 -L -o Merlin1.avi

rip DVD into low quality Xvid Avi (mono, low quality)

HandBrakeCLI -i /dev/dvd -E lame -B 64 -R 24 -6 mono -X 512 -e xvid -b 500 -t2 -o Systematisches_Vorgehen.avi

7.4.2 mencoder

extract and convert audio

avconv -i input.avi -vn -ar 44100 -ac 2 -ab 192 -f mp3 output.mp3

merge a video file with an audio file

mencoder -audiofile newaudio.wav -oac mp3lame -ovc copy videofile.avi -o new-videofile.avi

add additional audio track to video file (track 0 and 1 from input0, track 0 from input1)

avconv -i input0.m4v -i input1.mp3 -map 0:0 -map 0:1 -map 1:0 -vcodec copy -acodec copy output2.5.mkv

merge subtitles to a movie file

mkdir $HOME/.mplayer
cp /usr/share/fonts/truetype/...font.ttf $HOME/.mplayer/subfont.ttf
mencoder -o movie_output.avi -slang de -sub movie_input.srt -oac copy -ovc lavc -lavcopts vcodec=mpeg4:mbd=2:trell -idx movie_input.avi

7.5 audio handling / converting

lame -b 64 -mj --mp3input --resample 24 -h Silbermond\ -\ Das\ Beste.mp3 Silbermond\ -\ Das\ Beste-lq.mp3
for i in *; do lame -b 64 -mj --mp3input --resample 24 -h "$i" "../mplq/$i" ; done

echo -n Album:; read album
echo -n Artist:; read artist
for i in *; do id3tag -a"$artist" -A"$album" -s"$(basename $i .mp3)" "$i"; done

create iPhone Ringtone

# 1st create 30sec mp3 file for ringtone
ffmpeg -i PinkyBrain.mp3 -y PinkyBrain.m4a
mv PinkyBrain.m4a PinkyBrain.m4r
# then use it in itunes as ringtone

8 OTHER

8.1 extract all ansible-doc manuals into one single file

> ansible-alldoc.txt
for t in become cache callback cliconf connection httpapi inventory lookup netconf shell module strategy vars
do 
   ansible-doc -l -t $t | awk '{print $1}' | while read m
   do 
      echo "========== $t ========== $m ===========" >> ansible-alldoc.txt 
      ansible-doc -t $t $m >> ansible-alldoc.txt
   done
done

8.2 Delete Git project history

• Checkout

git checkout --orphan latest_branch

• Add all the files

git add -A

• Commit the changes

git commit -am "commit message"

• Delete the branch

git branch -D master

• Rename the current branch to master

git branch -m master

• Finally, force update your repository

git push -f origin master

8.3 Remove Mozilla Firefox Parentlock for multiple logins on same account

$HOME/.profile

rm -f $HOME/.mozilla/firefox/*/.parentlock

8.4 Assign mouse button action with xbindkeys

 # pacman -S xbindkeys xvkbd
 xbindkeys --defaults > $HOME/.xbindkeysrc

• scan for needed mouse code

 xbindkeys -k

• add it to configuration

vi $HOME/.xbindkeysrc
------
"skippy-xd"
  m:0x0 + b:9
------ 

• run config

xbindkeys

Do not forget to run when x11 starts