High IO Wait

for x in `seq 1 1 100`; do ps -eo state,pid,cmd | grep "^D"; echo "----"; sleep 5; done

ref: http://bencane.com/2012/08/troubleshooting-high-io-wait-in-linux/

#!/bin/sh

while true; do
    echo "---------- `date` ---------"; 
    uptime
    echo "#################"
    ps -eo state,pid,args | grep "^D"; 
    echo "#################"
    top -b -n 1 -c |head
    #iotop -o -b -n 1 | head
    echo "#################"
    grep Dirty /proc/meminfo
    grep -A 1 dirty /proc/vmstat
    sleep 10
done

see if the process does fsync

strace -p pid -e trace=fsync,fdatasync

monitor a process's cpu, rss and etc

monitor()
{
    pid=$1 peakRSS=0 peakCPU=0 RUNTIME=0 totWrite=0 totalCPU=0
    
    while true; do
        sampleRSS="$(ps -o rss= $pid 2> /dev/null)" || break
        let peakRSS='sampleRSS > peakRSS ? sampleRSS : peakRSS'

        sampleCPU="$(ps -o pcpu= $pid 2> /dev/null)" || break
        #echo $sampleCPU
        res=`echo "$sampleCPU > $peakCPU" | bc`
        if [ $res -eq 1 ]; then
            peakCPU=$sampleCPU
        fi

        totalCPU=`echo "$totalCPU + $sampleCPU" | bc`

        sampeWrite=`grep "^write_bytes" /proc/$pid/io | awk '{print $2}'`
        let totWrite='sampeWrite > totWrite ? sampeWrite : totWrite'

        RUNTIME=$(($RUNTIME + 1))
        sleep 1
    done
    avgcpu=`echo "scale = 2;  $totalCPU / $RUNTIME" | bc`
    echo "PeakRSS: $peakRSS" 
    echo "PeakCPU: $peakCPU" 
    echo "AvgCPU:  $avgcpu"
    echo "TotWrite: $totWrite" 
    echo "Runtime: $RUNTIME"
}