useful_commands.txt

# read file from remote server and display it in console
ssh host 'tail -f /var/log/application.log'
ssh -tA <username>@<bastion> ssh -o StrictHostKeyChecking=no <username>@<destination> 'tail -f /var/log/application.log'

# port forwarding over SSH (forwards local port 9000 to destination:9000 over bastion:9000)
ssh -L 9000:localhost:9000 -tA <username>@<bastion> ssh -L 9000:localhost:9000 -o StrictHostKeyChecking=no <username>@<destination> -N

# tip 0001: infinite loop
while true; do <PLACE COMMAND HERE>; sleep 1; done

# tip 0002: email logs in attachement
mutt -a abcdef.log mail@domain.com </dev/null

# tip 0003: zip file:
zip outputfilename.zip /input/file/name.ext

# tip 0004: install SMART monitoring tools on Mac
#	- install macports:
#	-- download from macports.org
#	-- compile && update ~/.bash_profile
sudo port install smartmontools

# tip 0005: lookup SMART info on Mac
#	- look at 233 Media_Wearout_Indicator 0x0032   099   099 (remaining life in %)
smartctl -a disk0

# download entire site (site mirror, site dump)
wget -mkEpnp http://example.org

# dig, nslookup, and host will not see entries in it because they bypass the system's resolver and do raw DNS
# lookups. They've always done this, so this is not new in Yosemite. The "official" way to do a lookup through the
# system resolver in OS X is to use dscacheutil
dscacheutil -q host -a name www.example.com

# get external IP address (any of this command)
curl http://ipecho.net/plain
curl http://ipinfo.io/ip
curl -s checkip.dyndns.org
curl ipinfo.io/185.85.220.193 # returns even geo location

# list of open ports
sudo lsof -i | egrep 'ESTABLISHED|LISTEN'

# installation date
sudo dumpe2fs /dev/sda5 | grep 'Filesystem created:'

# kernel install
sudo dpkg -i *.deb

# delete old kernels
sudo aptitude purge $(aptitude search ~ilinux-image -F %p|egrep -v "$(uname -r)|linux-image-generic")

# remove orphaned libs
sudo orphaner

# quick backup
rsync -avP --delete --exclude-from=do-not-backup.txt /Volumes/backup/ /Volumes/backup2 2>~/backup.log
sudo dd if=/dev/sourcedisk | pv -s 2T | sudo dd of=/dev/targetdisk conv=noerror,sync

# extract tar.bz2
tar -jxvf filename.tar.bz2

# extract tar.gz
tar -zxvf filename.tar.gz

# dependency tree (java, maven)
mvn dependency:tree

# unused modules (java, maven) -- sometimes works
mvn dependency:analyze

# find a file which contains given text (recursive, shows path to file):
grep -H -r "text" . | cut -d: -f1

# performs search using a file as an input for search criteria (each line in a file represents separate search criteria):
while read email; do echo Searching $email; grep -H -r $email --include "*.xml" . | cut -d: -f1; done < ~/temp/emails.txt

# search across all *.gz files
find -name \*.gz -print0 | xargs -0 zgrep "any text to find"

# counts matches in a file:
grep -c "pattern"

# counts matches in all files:
grep -r -H -c "pattern" .

# counts matches in all files (sum of all values in all files): 
grep -r -H -c "== null" . | cut -d ":" -f2 | grep [1-9][0-9]* | awk 'BEGIN {c=0}{c=c+$0} END {print c}'

# search for "non-existing" resources, counts number of calls:
egrep -o "Uri (.*?) does not exist" fo-web.log | sort | uniq -c | sort -nr > links.txt 

# ZIP all files that with matched filenames:
find . -noleaf -type f -print | egrep "2013-07-18" | sed 's/ /\\ /g' | xargs zip /folder/2013-07-18/my.zip

# find and copy files:
find . -noleaf -type f -name "*.xml" | egrep "(foo|bar)/2013-07-18" | sed 's/ /\\ /g' | xargs cp -t /folder/foo-or-bar

# who and when was logged in:
last

# setup of java-monitoring on remote host for visualvm:
# note: it requires jstatd file 
# [user@server jdk1.8.0_120]$ cat /usr/local/j2sdk/.jstatd.all.policy
# grant codebase "file:/usr/local/j2sdk/lib/tools.jar" {
#    permission java.security.AllPermission;
# };
# jstatd has to be executed under the same user which will start a server with an application
jstatd -J-Djava.security.policy=/usr/local/j2sdk/.jstatd.all.policy &

# JAVA: get heap dump of java application in docker container (1 is a process ID)
docker exec <container-name> jcmd 1 GC.heap_dump /tmp/docker.hprof

# JAVA: flags for performance testing. Use in conjunction with "jmc"
-XX:+UnlockCommercialFeatures
-XX:+FlightRecorder
-XX:+UnlockDiagnosticVMOptions
-XX:+DebugNonSafepoints

# JAVA: dump on OOM errors
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/path/to/dump

# for loop
for i in {1..10}; do echo $i; done

# downgrade python tools
sudo pip3 install stups-senza==1.0.92

# find a  list of versions for downgrade
sudo pip3 install stups-senza=abracadabra

# simply HTTP server
python -m SimpleHTTPServer

# clone git repo with all branches and tags
git branch -r | grep -v '\->' | while read remote; do git branch --track "${remote#origin/}" "$remote"; done
git fetch --all
git fetch origin 'refs/tags/*:refs/tags/*'
git pull --all
git remote add new_origin https://github.com/user/repo.git
git push --all new_origin
git push 'refs/tags/*:refs/tags/*' new_origin

# quickly create big file / allocate a lot of space
fallocate -l 100G big.file

# determine file type
file your_file.raw

# download access logs from AWS
aws s3 cp s3://bucket-with-logs/sub/folder . --recursive

# upload files to S3 + cache control
aws s3 cp folder s3://bucket/folder --recursive --cache-control max-age=0,no-cache,no-store,must-revalidate

# catch 4xx & 5xx + connections closed by client (-1s):
egrep "([0-9]{1,2}.[0-9]{3}\s|-1\s){3}(200\s)?[4-5][0-9]{2}" -r --include "*.log" .

# catch slow queries from logs:
egrep "(\s[1-9]{1}[0-9]{0,2}\.[0-9]{3}\s)" -r --include "*.log" .

# invalid requests or connections closed by client:
egrep "\s-1\s" -r --include "*.log" . | wc -l

# generate strong password
head -c24 /dev/urandom | base64 gives 144 bits of entropy

# SCALA
########
# to debug scala implicits
import language.implicitConversions
import scala.reflect.runtime.universe.reify
reify { implicitly[Ordering[Int]] } // here you see explanation

# Apple disk encryption progress
diskutil apfs list
diskutil cs list

# Count number of code lines in git repository per user
git ls-files | while read f; do git blame -w -M -C -C --line-porcelain "$f" | grep -I '^author '; done | sort -f | uniq -ic | sort -n

# Count number of lines changed by user since specific date
git log --shortstat --author "Dmitry Krivaltsevich" --since "yesterday" | grep "files changed" | awk '{files+=$1; inserted+=$4; deleted+=$6} END {print "files changed:”, files, "lines inserted:", inserted, "lines deleted:", deleted}'