lugh.ch

as nerdy as needed.

Useful Linux, Solaris and OS X commands


Useful commands

disk usage

Ordered by size, descending

du -hax /var | sort -rh

vim

Get rid of all control characters

:%!tr -cd '[:print:]\n'

curl

URL shortening on command line

curl https://www.googleapis.com/urlshortener/v1/url -H 'Content-Type: application/json' -d '{"longUrl": "http://example.org"}'

Check if your webserver supports gzip compression with curl

curl -I -H "Accept-Encoding: gzip,deflate" http://example.org

curl: verify certificates

Using curl -k is a no-brainer, but really defeats the purpose of encryption. If you want to permanently store certificates as trusted, here's how to do it on Debian. This also works when using libcurl with PHP.

openssl s_client -connect example.org:443 # Get remote certificate
# Save the certificate in /usr/local/share/ca-certificates/example.crt
update-ca-certificates

Varnish

Filter varnishlog to show all relevant content of XID

varnishlog -d -m TxHeader:<XID number> | awk '$1 !~ /0/ { print $0 }'

Varnish 4: filter by header

varnishlog -b -q "ReqHeader eq 'Host: example.org'"

Varnish 3: filter by URL

Filter for a frontend request (client => Varnish)

varnishlog -c -m RxURL:"/api/rest/products"

Filter for a backend request (Varnish => backend)

varnishlog -b -m TxURL:"/api/rest/products"

Check config file for errors

varnishd -C -f /etc/varnish/default.vcl

Show URLs hitting the backend the most

varnishtop -i txurl

Numeric permission list of all directories/files

Can be useful to restore in case of a fatal chmod -R 777 /.

find / | xargs stat -c 'chmod %a "'%n'"'

Apache: show compiled and shared modules loaded in Apache

/usr/bin/httpd -t -D DUMP_MODULES

Highlight changing network information (rx,tx etc.)

watch -n 2 -d '/sbin/ifconfig eth0'

Start xen guest and attach to virtual console

To access grub and see bootup messages

xm create -c vmname

Package managers

Which package contains that file?

# Red Hat
rpm -qf /bin/mount
# Debian
dpkg -S /bin/mount

Which packages were installed lately?

# Red Hat 
rpm -qa --last | tac
# Debian
grep -E '(UPGRADE|INSTALL)' /var/log/aptitude

Which not installed package provides the specified file?

# Red Hat
yum whatprovides /bin/traceroute

Text utilities (sed, grep, awk etc.)

Replace in-file (or in-place edit) with Perl

perl -pi -e 's/oldtext/newtext/' file.txt

Delete a block of text with sed

sed "/startOfBlock/,/endOfBlock/d" file.txt

Pad single-digit fields in MAC addresses

Thanks to Yannick Denzer

echo "a:0:1:0:a:43" | sed -E 's/[^:]+/0&/g;s/[^:]([^:][^:])/\1/g'

LDAP

ldapsearch with TLS

ldapsearch -x -Z <ldapserver>

ldapsearch SSL against Active Directory (AD)

Requires TLS_REQCERT never in /etc/openldap/ldap.conf.

ldapsearch -x -LLL -D "binduser" -w "bindpw" -b "dc=corp,dc=example,dc=org" -H ldaps://xxx.xx.xx.xx -v

Capturing traffic with tcpdump

tcpdump -i eth3 tcp port 389 -w /root/tcpdump.txt

Open reverse SSH tunnel

ssh -L localport:destination:port user@gateway

MySQL

Show extended table info like collation, privileges etc.

SHOW FULL COLUMNS FROM tblname;

Maintenance

mysqlcheck -u root --auto-repair --check --optimize --all-databases

Show database engine used

SHOW TABLE STATUS FROM `db_name`;

Calculate size of all databases

SELECT table_schema AS "Database name",
SUM(data_length + index_length) / 1024 / 1024 AS "Size (MB)"
FROM information_schema.TABLES
GROUP BY table_schema;

Calculate size of all tables in a database

SELECT table_name AS "Table",
ROUND(((data_length + index_length) / 1024 / 1024), 2) AS size
FROM information_schema.TABLES
WHERE table_schema = "yourdatabase"
ORDER BY size DESC;

Visualize import progress

Requires the "pv" utility.

pv -i 1 -p -t -e /tmp/dump.sql | mysql -u foo -pXXXXX database

Import/export directly from/to compressed (gzip) file

# Export:
mysqldump -u user -p database | gzip > database.sql.gz

# Import:
gunzip < database.sql.gz | mysql -u user -p database

Kill idle (Sleep) processes

mysqladmin proc | grep "<user>.*<db>.*Sleep" | sort -r -n -k6 | awk {'print $2;'} | tr -s '\n' ',' | xargs mysqladmin kill

PostgreSQL

Dump DB to remote file via SSH

pg_dump -U pg-user database | ssh user@192.168.1.1 "cat - > /var/dump.sql"

Get database size

SELECT pg_size_pretty(pg_database_size('databasename')) as fulldbsize;

Hardware related

XSCF: Setting a route

setroute -c add -n 0.0.0.0 -g <gateway-ip> xscf#0-lan#1

Read hardware sensors from Sun server

ipmitool -v -U username -H 192.168.1.1 sdr list all

Solaris

zone commands

zoneadm list -iv # on global zone, list all zones

Determine which package a file belongs to

pkgchk -l -p /path/to/file
grep filename /var/sadm/install/contents

OS X

MacPorts: upate ports tree and upgrade packages

port selfupdate && port upgrade outdated

Debian: hold a package to prevent updating

echo linux-image-2.6-686-bigmem hold | dpkg --set-selections

Get IP address from ifconfig output

# Linux
ifconfig eth0 | awk -F ' *|:' '/inet addr/{print $4}'
# OS X
ifconfig en1 | awk -F ' *|:' '/inet /{print $2}'

Convert UNIX timestamp to an human-readable format

date -d @1305547782

mpd: play next song matching ‘infernal war’

while true; do mpc next | grep -qi 'infernal war' && break; done

Put init script to standard runlevels

# Debian
update-rc.d nagios defaults
# RHEL
chkconfig on ntpd

Sort output by column (separator = $IFS)

ps aux | sort -nk 6

Convert .ts (MPEG-TS) files

mencoder sourcefile.ts -oac mp3lame -ovc lavc -lavcopts aspect=16/9 -o out.avi

OpenSSL

Display fingerprint of a certificate. "type" is md5, sha1, sha256 etc.

openssl x509 -fingerprint -noout -in newcert.pem -<type>

Get certificate for standard SSL-only connections

openssl s_client -connect foo.example.org:443

Get certificate for STARTTLS services

openssl s_client -connect foo.example.org:25 -starttls smtp -CApath /etc/ssl/certs
openssl s_client -connect mail.example.org:143 -starttls imap -CApath /etc/ssl/certs

Display local certificate details

openssl x509 -in /path/to/cert -text

Convert PKCS#7 certificate to PEM format

openssl pkcs7 -in pkcs7.file -text -out cert.pem -print_certs

Convert PKCS#12 to PEM

Host certificate:

openssl pkcs12 -in host.domain.p12 -clcerts -nokeys -out host.domain.cert.pem
openssl pkcs12 -in host.domain.p12 -nocerts -nodes -out host.domain.key.pem

User certificate:

openssl pkcs12 -in export.p12 -clcerts -nokeys -out cert.pem
openssl pkcs12 -in export.p12 -nocerts -out key.pem

256 color terminal stuff

tput colors                 # get colors
export TERM=xterm-256colors # if installed
xrdb -load $HOME/.Xdefaults # to activate color changes in X terminals

Horde Webmail with IMAP proxy, thread sort error

If you encounter errors when opening folders sorted in a special way (by threads for example), just run (Horde Groupware 4+):

horde-clear-cache

SGD connection fails with "Failed to install SGD Client" on Debian 6 (64bit)

I was missing some 32-bit libraries. Install the ia32-libs package, then the problem should besolved.

user@host:~$ ldd .tarantella/tcc/4.50.937/ttatcc 
linux-gate.so.1 =>  (0xf77d5000)
libX11.so.6 => /usr/lib32/libX11.so.6 (0xf769e000)
libXmu.so.6 => /usr/lib32/libXmu.so.6 (0xf7688000)
libXt.so.6 => /usr/lib32/libXt.so.6 (0xf7635000)
libXext.so.6 => /usr/lib32/libXext.so.6 (0xf7626000)
libSM.so.6 => /usr/lib32/libSM.so.6 (0xf761e000)
libICE.so.6 => /usr/lib32/libICE.so.6 (0xf7607000)
libpthread.so.0 => /lib32/libpthread.so.0 (0xf75ee000)
libdl.so.2 => /lib32/libdl.so.2 (0xf75ea000)
libgcc_s.so.1 => /usr/lib32/libgcc_s.so.1 (0xf75cb000)
libc.so.6 => /lib32/libc.so.6 (0xf7484000)
/lib/ld-linux.so.2 (0xf77d6000)
libxcb.so.1 => /usr/lib32/libxcb.so.1 (0xf746b000)
libuuid.so.1 => /lib32/libuuid.so.1 (0xf7467000)
libXau.so.6 => /usr/lib32/libXau.so.6 (0xf7464000)
libXdmcp.so.6 => /usr/lib32/libXdmcp.so.6 (0xf745e000)

Scan SCSI bus for new harddisks

If fdisk doesn't see new disks even after a partprobe, issue:

echo "- - -" > /sys/class/scsi_host/host#/scan

Show I/O operations of all drives

Raise number 5 at the beginning for more accurate results.

iostat 1 5 -xdnN | egrep "[a-zA-Z].*[0-9]\.[0-9][0-9][[:space:]]" | awk {'if ($1 ~ /:\//) print $9,$10,$1; else print $4,$5,$1'} | tail -n +2

Indent an unformatted XML file

xmlstarlet fo --indent-tab --omit-decl foo.xml

Python/Django: Create A-Z index in template

Source: http://stackoverflow.com/questions/3617041/a-z-index-django

{% ifchanged food.name.0 %} <h1>{{food.name.0}}</h1>{% endifchanged %}

DNS: query SRV records

Useful for Jabber:

dig SRV _xmpp-client._tcp.example.com
dig SRV _xmpp-server._tcp.example.com

Nagios/Icinga: convert timestamps in nagios.log

perl -pe 's/(\d+)/localtime($1)/e' /usr/local/nagios/var/nagios.log | tail -20

Kill and logout Linux shell user

The "skill" command is in the package "procps":

skill -KILL -u username

Linux ACL

Backup and restore Linux ACLs recursively

getfacl -R /dir/with/acls > /tmp/bkp.acl
setfacl --restore=/tmp/bkp.acl --test # omit --test if all is OK

Copy ACL from file/directory A to file/directory B

getfacl file1 | setfacl --set-file=- file2

tar file via SSH

tar cfzp - /dir/to/backup | ssh root@192.168.1.2 "cat > /tmp/destination.tar.gz"

SMTP: Test mail throughput with Postfix tools

To test the theoretical capable mail volume, use smtp-sink on the destination, which acts as an SMTP blackhole that accepts mail and throws it away. On the sender side, use smtp-source as a bulk mailer with various config options like parallel sessions, mail size etc.

# Destination:
smtp-sink -c -u postfix -M 10000 0.0.0.0:25 1024

# Source:
time smtp-source -s 20 -l 30000 -m 10000 -c -f sender@example.org -t receiver@example.org smtp-sink-address:25

Speed up Software RAID (md) resync

If one of your RAID devices has failed, you might be able to speed up the recovery (values are in Kb/s):

Check the current resync speed:

grep speed /proc/mdstat
[===>.................]  recovery = 19.6% (191587648/975193600) finish=459.7min speed=28404K/sec

Check the current minimal/max bandwidth defined:

cat /proc/sys/dev/raid/speed_limit_max
20000
cat /proc/sys/dev/raid/speed_limit_min
5000

Raise min/max

echo 80000 > /proc/sys/dev/raid/speed_limit_max
echo 40000 > /proc/sys/dev/raid/speed_limit_min

El-cheapo way to measure the runtime of a program

while [ $(ps -p <PID> -o etime= | grep -c ".*") -gt 0 ]; do echo "$(ps -p <PID> -o etime=)" >> /tmp/runtime.log; sleep 30; done

Git

Rename a tag

Source: http://stackoverflow.com/questions/1028649/rename-a-tag-in-git

git tag new old
git tag -d old
git push origin :refs/tags/old
git push --tags

Nice and ionice together

nice -n 19 ionice -c2 -n7 <command>

dd, write ISO to disk

cat centos64.img | ssh -C root@x.x.x.xxx "dd of=/dev/sda bs=1M"

nmap

Ping hosts in a subnet

nmap -sP 192.168.1.1-254

Bacula

Reports (query.sql)

:Monthly report (current month)
SELECT JobId,StartTime,EndTime,Name,Level,JobErrors AS Errors FROM Job LEFT JOIN Status on Job.JobStatus=Status.JobStatus WHERE StartTime > DATE_FORMAT(now() - INTERVAL 1 MONTH, '%Y-%m-%d') ORDER BY StartTime ASC;

:Monthly report (previous month) 
SELECT JobId,StartTime,EndTime,Name,Level,JobErrors AS Errors FROM Job LEFT JOIN Status on Job.JobStatus=Status.JobStatus WHERE StartTime > DATE_FORMAT(now() - INTERVAL 2 MONTH, '%Y-%m-%d') AND StartTime < DATE_FORMAT(now() - INTERVAL 3 MONTH, '%Y-%m-%d') ORDER BY StartTime ASC;

:Backup size (Full)                                                           
SELECT Client.Name, ROUND(AVG(Job.JobBytes)/1024/1024/1024, 3) AS "Average GB", ROUND(STDDEV(Job.JobBytes)/1024/1024/1024, 3) AS "Standard Deviation GB" FROM Job INNER JOIN Client ON Job.ClientId=Client.ClientId WHERE Level = 'F' GROUP BY Client.Name ORDER BY "Average GB" DESC;

:Backup size (Incremental)                                                    
SELECT Client.Name, ROUND(AVG(Job.JobBytes)/1024/1024/1024, 3) AS "Average GB", ROUND(STDDEV(Job.JobBytes)/1024/1024/1024, 3) AS "Standard Deviation GB" FROM Job INNER JOIN Client ON Job.ClientId=Client.ClientId WHERE Level = 'I' GROUP BY Client.Name ORDER BY "Average GB" DESC;