Useful commands
disk usage
Ordered by size, descending
du -hax /var | sort -rh
vim
Get rid of all control characters
:%!tr -cd '[:print:]\n'
curl
URL shortening on command line
curl https://www.googleapis.com/urlshortener/v1/url -H 'Content-Type: application/json' -d '{"longUrl": "http://example.org"}'
Check if your webserver supports gzip compression with curl
curl -I -H "Accept-Encoding: gzip,deflate" http://example.org
curl: verify certificates
Using curl -k is a no-brainer, but really defeats the purpose of encryption. If you want to permanently store certificates as trusted, here's how to do it on Debian. This also works when using libcurl with PHP.
openssl s_client -connect example.org:443 # Get remote certificate
# Save the certificate in /usr/local/share/ca-certificates/example.crt
update-ca-certificates
Varnish
Filter varnishlog to show all relevant content of XID
varnishlog -d -m TxHeader:<XID number> | awk '$1 !~ /0/ { print $0 }'
Varnish 4.x: filter based on header
varnishlog -b -q "ReqHeader eq 'Host: example.org'"
Check config file for errors
varnishd -C -f /etc/varnish/default.vcl
Show URLs hitting the backend the most
varnishtop -i txurl
Numeric permission list of all directories/files
Can be useful to restore in case of a fatal chmod -R 777 /
.
find / | xargs stat -c 'chmod %a "'%n'"'
Apache: show compiled and shared modules loaded in Apache
/usr/bin/httpd -t -D DUMP_MODULES
Highlight changing network information (rx,tx etc.)
watch -n 2 -d '/sbin/ifconfig eth0'
Start xen guest and attach to virtual console
To access grub and see bootup messages
xm create -c vmname
Package managers
Which package contains that file?
# Red Hat
rpm -qf /bin/mount
# Debian
dpkg -S /bin/mount
Which packages were installed lately?
# Red Hat
rpm -qa --last | tac
# Debian
grep -E '(UPGRADE|INSTALL)' /var/log/aptitude
Which not installed package provides the specified file?
# Red Hat
yum whatprovides /bin/traceroute
Text utilities (sed, grep, awk etc.)
Replace in-file (or in-place edit) with Perl
perl -pi -e 's/oldtext/newtext/' file.txt
Delete a block of text with sed
sed "/startOfBlock/,/endOfBlock/d" file.txt
Pad single-digit fields in MAC addresses
Thanks to Yannick Denzer
echo "a:0:1:0:a:43" | sed -E 's/[^:]+/0&/g;s/[^:]([^:][^:])/\1/g'
LDAP
ldapsearch with TLS
ldapsearch -x -Z <ldapserver>
ldapsearch SSL against Active Directory (AD)
Requires TLS_REQCERT never in /etc/openldap/ldap.conf.
ldapsearch -x -LLL -D "binduser" -w "bindpw" -b "dc=corp,dc=example,dc=org" -H ldaps://xxx.xx.xx.xx -v
Capturing traffic with tcpdump
tcpdump -i eth3 tcp port 389 -w /root/tcpdump.txt
Open reverse SSH tunnel
ssh -L localport:destination:port user@gateway
MySQL
Show extended table info like collation, privileges etc.
SHOW FULL COLUMNS FROM tblname;
Maintenance
mysqlcheck -u root --auto-repair --check --optimize --all-databases
Show database engine used
SHOW TABLE STATUS FROM `db_name`;
Calculate size of all databases
SELECT table_schema AS "Database name", SUM(data_length + index_length) / 1024 / 1024 AS "Size (MB)" FROM information_schema.TABLES GROUP BY table_schema;
Calculate size of all tables in a database
SELECT table_name AS "Table", ROUND(((data_length + index_length) / 1024 / 1024), 2) AS size FROM information_schema.TABLES WHERE table_schema = "celum" ORDER BY size DESC;
Visualize import progress
Requires the "pv" utility.
pv -i 1 -p -t -e /tmp/dump.sql | mysql -u foo -pXXXXX database
Import/export directly from/to compressed (gzip) file
# Export:
mysqldump -u user -p database | gzip > database.sql.gz
# Import:
gunzip < database.sql.gz | mysql -u user -p database
Kill idle (Sleep) processes
mysqladmin proc | grep "<user>.*<db>.*Sleep" | sort -r -n -k6 | awk {'print $2;'} | tr -s '\n' ',' | xargs mysqladmin kill
PostgreSQL
Dump DB to remote file via SSH
pg_dump -U pg-user database | ssh user@192.168.1.1 "cat - > /var/dump.sql"
Get database size
SELECT pg_size_pretty(pg_database_size('databasename')) as fulldbsize;
Hardware related
XSCF: Setting a route
setroute -c add -n 0.0.0.0 -g <gateway-ip> xscf#0-lan#1
Read hardware sensors from Sun server
ipmitool -v -U username -H 192.168.1.1 sdr list all
Solaris
zone commands
zoneadm list -iv # on global zone, list all zones
Determine which package a file belongs to
pkgchk -l -p /path/to/file
grep filename /var/sadm/install/contents
OS X
MacPorts: upate ports tree and upgrade packages
port selfupdate && port upgrade outdated
Debian: hold a package to prevent updating
echo linux-image-2.6-686-bigmem hold | dpkg --set-selections
Get IP address from ifconfig output
# Linux
ifconfig eth0 | awk -F ' *|:' '/inet addr/{print $4}'
# OS X
ifconfig en1 | awk -F ' *|:' '/inet /{print $2}'
Convert UNIX timestamp to an human-readable format
date -d @1305547782
mpd: play next song matching ‘infernal war’
while true; do mpc next | grep -qi 'infernal war' && break; done
Put init script to standard runlevels
# Debian
update-rc.d nagios defaults
# RHEL
chkconfig on ntpd
Sort output by column (separator = $IFS)
ps aux | sort -nk 6
Convert .ts (MPEG-TS) files
mencoder sourcefile.ts -oac mp3lame -ovc lavc -lavcopts aspect=16/9 -o out.avi
OpenSSL
Display fingerprint of a certificate. "type" is md5, sha1, sha256 etc.
openssl x509 -fingerprint -noout -in newcert.pem -<type>
Get certificate for standard SSL-only connections
openssl s_client -connect foo.example.org:443
Get certificate for STARTTLS services
openssl s_client -connect foo.example.org:25 -starttls smtp -CApath /etc/ssl/certs
openssl s_client -connect mail.example.org:143 -starttls imap -CApath /etc/ssl/certs
Display local certificate details
openssl x509 -in /path/to/cert -text
Convert PKCS#7 certificate to PEM format
openssl pkcs7 -in pkcs7.file -text -out cert.pem -print_certs
Convert PKCS#12 to PEM
Host certificate:
openssl pkcs12 -in host.domain.p12 -clcerts -nokeys -out host.domain.cert.pem
openssl pkcs12 -in host.domain.p12 -nocerts -nodes -out host.domain.key.pem
User certificate:
openssl pkcs12 -in export.p12 -clcerts -nokeys -out cert.pem
openssl pkcs12 -in export.p12 -nocerts -out key.pem
256 color terminal stuff
tput colors # get colors
export TERM=xterm-256colors # if installed
xrdb -load $HOME/.Xdefaults # to activate color changes in X terminals
Horde Webmail with IMAP proxy, thread sort error
If you encounter errors when opening folders sorted in a special way (by threads for example), just run (Horde Groupware 4+):
horde-clear-cache
SGD connection fails with "Failed to install SGD Client" on Debian 6 (64bit)
I was missing some 32-bit libraries. Install the ia32-libs package, then the problem should besolved.
user@host:~$ ldd .tarantella/tcc/4.50.937/ttatcc
linux-gate.so.1 => (0xf77d5000)
libX11.so.6 => /usr/lib32/libX11.so.6 (0xf769e000)
libXmu.so.6 => /usr/lib32/libXmu.so.6 (0xf7688000)
libXt.so.6 => /usr/lib32/libXt.so.6 (0xf7635000)
libXext.so.6 => /usr/lib32/libXext.so.6 (0xf7626000)
libSM.so.6 => /usr/lib32/libSM.so.6 (0xf761e000)
libICE.so.6 => /usr/lib32/libICE.so.6 (0xf7607000)
libpthread.so.0 => /lib32/libpthread.so.0 (0xf75ee000)
libdl.so.2 => /lib32/libdl.so.2 (0xf75ea000)
libgcc_s.so.1 => /usr/lib32/libgcc_s.so.1 (0xf75cb000)
libc.so.6 => /lib32/libc.so.6 (0xf7484000)
/lib/ld-linux.so.2 (0xf77d6000)
libxcb.so.1 => /usr/lib32/libxcb.so.1 (0xf746b000)
libuuid.so.1 => /lib32/libuuid.so.1 (0xf7467000)
libXau.so.6 => /usr/lib32/libXau.so.6 (0xf7464000)
libXdmcp.so.6 => /usr/lib32/libXdmcp.so.6 (0xf745e000)
Scan SCSI bus for new harddisks
If fdisk doesn't see new disks even after a partprobe, issue:
echo "- - -" > /sys/class/scsi_host/host#/scan
Show I/O operations of all drives
Raise number 5 at the beginning for more accurate results.
iostat 1 5 -xdnN | egrep "[a-zA-Z].*[0-9]\.[0-9][0-9][[:space:]]" | awk {'if ($1 ~ /:\//) print $9,$10,$1; else print $4,$5,$1'} | tail -n +2
Indent an unformatted XML file
xmlstarlet fo --indent-tab --omit-decl foo.xml
Python/Django: Create A-Z index in template
Source: http://stackoverflow.com/questions/3617041/a-z-index-django
\{% ifchanged food.name.0 %} <h1>\{{food.name.0}}</h1>\{% endifchanged %}
DNS: query SRV records
Useful for Jabber:
dig SRV _xmpp-client._tcp.example.com
dig SRV _xmpp-server._tcp.example.com
Nagios/Icinga: convert timestamps in nagios.log
perl -pe 's/(\d+)/localtime($1)/e' /usr/local/nagios/var/nagios.log | tail -20
Kill and logout Linux shell user
The "skill" command is in the package "procps":
skill -KILL -u username
Linux ACL
Backup and restore Linux ACLs recursively
getfacl -R /dir/with/acls > /tmp/bkp.acl
setfacl --restore=/tmp/bkp.acl --test # omit --test if all is OK
Copy ACL from file/directory A to file/directory B
getfacl file1 | setfacl --set-file=- file2
tar file via SSH
tar cfzp - /dir/to/backup | ssh root@192.168.1.2 "cat > /tmp/destination.tar.gz"
SMTP: Test mail throughput with Postfix tools
To test the theoretical capable mail volume, use smtp-sink on the destination, which acts as an SMTP blackhole that accepts mail and throws it away. On the sender side, use smtp-source as a bulk mailer with various config options like parallel sessions, mail size etc.
# Destination:
smtp-sink -c -u postfix -M 10000 0.0.0.0:25 1024
# Source:
time smtp-source -s 20 -l 30000 -m 10000 -c -f sender@example.org -t receiver@example.org smtp-sink-address:25
Speed up Software RAID (md) resync
If one of your RAID devices has failed, you might be able to speed up the recovery (values are in Kb/s):
Check the current resync speed:
grep speed /proc/mdstat
[===>.................] recovery = 19.6% (191587648/975193600) finish=459.7min speed=28404K/sec
Check the current minimal/max bandwidth defined:
cat /proc/sys/dev/raid/speed_limit_max
20000
cat /proc/sys/dev/raid/speed_limit_min
5000
Raise min/max
echo 80000 > /proc/sys/dev/raid/speed_limit_max
echo 40000 > /proc/sys/dev/raid/speed_limit_min
El-cheapo way to measure the runtime of a program
while [ $(ps -p <PID> -o etime= | grep -c ".*") -gt 0 ]; do echo "$(ps -p <PID> -o etime=)" >> /tmp/runtime.log; sleep 30; done
Git
Rename a tag
Source: http://stackoverflow.com/questions/1028649/rename-a-tag-in-git
git tag new old
git tag -d old
git push origin :refs/tags/old
git push --tags
Nice and ionice together
nice -n 19 ionice -c2 -n7 <command>
dd, write ISO to disk
cat centos64.img | ssh -C root@x.x.x.xxx "dd of=/dev/sda bs=1M"
nmap
Ping hosts in a subnet
nmap -sP 192.168.1.1-254
Bacula
Reports (query.sql)
:Monthly report (current month)
SELECT JobId,StartTime,EndTime,Name,Level,JobErrors AS Errors FROM Job LEFT JOIN Status on Job.JobStatus=Status.JobStatus WHERE StartTime > DATE_FORMAT(now() - INTERVAL 1 MONTH, '%Y-%m-%d') ORDER BY StartTime ASC;
:Monthly report (previous month)
SELECT JobId,StartTime,EndTime,Name,Level,JobErrors AS Errors FROM Job LEFT JOIN Status on Job.JobStatus=Status.JobStatus WHERE StartTime > DATE_FORMAT(now() - INTERVAL 2 MONTH, '%Y-%m-%d') AND StartTime < DATE_FORMAT(now() - INTERVAL 3 MONTH, '%Y-%m-%d') ORDER BY StartTime ASC;
:Backup size (Full)
SELECT Client.Name, ROUND(AVG(Job.JobBytes)/1024/1024/1024, 3) AS "Average GB", ROUND(STDDEV(Job.JobBytes)/1024/1024/1024, 3) AS "Standard Deviation GB" FROM Job INNER JOIN Client ON Job.ClientId=Client.ClientId WHERE Level = 'F' GROUP BY Client.Name ORDER BY "Average GB" DESC;
:Backup size (Incremental)
SELECT Client.Name, ROUND(AVG(Job.JobBytes)/1024/1024/1024, 3) AS "Average GB", ROUND(STDDEV(Job.JobBytes)/1024/1024/1024, 3) AS "Standard Deviation GB" FROM Job INNER JOIN Client ON Job.ClientId=Client.ClientId WHERE Level = 'I' GROUP BY Client.Name ORDER BY "Average GB" DESC;