1、top 查看cpu使用过高的PID

2、脚本 + Pid 查看该进程的堆栈信息

show-busy-java-threads  523938

查看到信息就可以发送给开发们看了

3、编写脚本附加x权限

vim /usr/local/bin/show-busy-java-threads

#!/bin/bash
# @Function
# Find out the highest cpu consumed threads of java processes, and print the stack of these threads.
#
# @Usage
#   $ ./show-busy-java-threads
#
# @online-doc https://github.com/oldratlee/useful-scripts/blob/master/docs/java.md#-show-busy-java-threads
# @author Jerry Lee (oldratlee at gmail dot com)
# @author superhj1987 (superhj1987 at 126 dot com)readonly PROG="`basename $0`"
readonly -a COMMAND_LINE=("$0" "$@")
# Get current user name via whoami command
#   See https://www.lifewire.com/current-linux-user-whoami-command-3867579
# Because if run command by `sudo -u`, env var $USER is not rewritten/correct, just inherited from outside!
readonly USER="`whoami`"################################################################################
# util functions
################################################################################# NOTE: $'foo' is the escape sequence syntax of bash
readonly ec=$'\033' # escape char
readonly eend=$'\033[0m' # escape endcolorEcho() {local color=$1shift# if stdout is console, turn on color output.[ -t 1 ] && echo "$ec[1;${color}m$@$eend" || echo "$@"
}colorPrint() {local color=$1shiftcolorEcho "$color" "$@"[ -n "$append_file" -a -w "$append_file" ] && echo "$@" >> "$append_file"[ -n "$store_dir" -a -w "$store_dir" ] && echo "$@" >> "${store_file_prefix}$PROG"
}normalPrint() {echo "$@"[ -n "$append_file" -a -w "$append_file" ] && echo "$@" >> "$append_file"[ -n "$store_dir" -a -w "$store_dir" ] && echo "$@" >> "${store_file_prefix}$PROG"
}redPrint() {colorPrint 31 "$@"
}greenPrint() {colorPrint 32 "$@"
}yellowPrint() {colorPrint 33 "$@"
}bluePrint() {colorPrint 36 "$@"
}die() {redPrint "Error: $@" 1>&2exit 1
}logAndRun() {echo "$@"echo"$@"
}logAndCat() {echo "$@"echocat
}usage() {local -r exit_code="$1"shift[ -n "$exit_code" -a "$exit_code" != 0 ] && local -r out=/dev/stderr || local -r out=/dev/stdout(( $# > 0 )) && { echo "$@"; echo; } > $out> $out cat <<EOF
Usage: ${PROG} [OPTION]... [delay [count]]
Find out the highest cpu consumed threads of java processes,
and print the stack of these threads.
Example:${PROG}       # show busy java threads info${PROG} 1     # update every 1 second, (stop by eg: CTRL+C)${PROG} 3 10  # update every 3 seconds, update 10 times
Output control:-p, --pid <java pid>      find out the highest cpu consumed threads fromthe specified java process.default from all java process.-c, --count <num>         set the thread count to show, default is 5.-a, --append-file <file>  specifies the file to append output as log.-S, --store-dir <dir>     specifies the directory for storingthe intermediate files, and keep files.default store intermediate files at tmp dir,and auto remove after run. use this option to keepfiles so as to review jstack/top/ps output later.delay                     the delay between updates in seconds.count                     the number of updates.delay/count arguments imitates the style ofvmstat command.
jstack control:-s, --jstack-path <path>  specifies the path of jstack command.-F, --force               set jstack to force a thread dump. use when jstackdoes not respond (process is hung).-m, --mix-native-frames   set jstack to print both java and native frames(mixed mode).-l, --lock-info           set jstack with long listing.prints additional information about locks.
CPU usage calculation control:-d, --top-delay           specifies the delay between top samples.default is 0.5 (second). get thread cpu percentageduring this delay interval.more info see top -d option. eg: -d 1 (1 second).-P, --use-ps              use ps command to find busy thread(cpu usage)instead of top command.default use top command, because cpu usage ofps command is expressed as the percentage oftime spent running during the *entire lifetime*of a process, this is not ideal in general.
Miscellaneous:-h, --help                display this help and exit.
EOFexit $exit_code
}################################################################################
# Check os support
################################################################################uname | grep '^Linux' -q || die "$PROG only support Linux, not support `uname` yet!"################################################################################
# parse options
################################################################################# NOTE: ARGS can not be declared as readonly!!
# readonly declaration make exit code of assignment to be always 0, aka. the exit code of `getopt` in subshell is discarded.
#   tested on bash 4.2.46
ARGS=`getopt -n "$PROG" -a -o p:c:a:s:S:Pd:Fmlh -l count:,pid:,append-file:,jstack-path:,store-dir:,use-ps,top-delay:,force,mix-native-frames,lock-info,help -- "$@"`
[ $? -ne 0 ] && { echo; usage 1; }
eval set -- "${ARGS}"while true; docase "$1" in-c|--count)count="$2"shift 2;;-p|--pid)pid="$2"shift 2;;-a|--append-file)append_file="$2"shift 2;;-s|--jstack-path)jstack_path="$2"shift 2;;-S|--store-dir)store_dir="$2"shift 2;;-P|--use-ps)use_ps=trueshift;;-d|--top-delay)top_delay="$2"shift 2;;-F|--force)force=-Fshift;;-m|--mix-native-frames)mix_native_frames=-mshift;;-l|--lock-info)more_lock_info=-lshift;;-h|--help)usage;;--)shiftbreak;;esac
donecount=${count:-5}update_delay=${1:-0}
[ -z "$1" ] && update_count=1 || update_count=${2:-0}
(( update_count < 0 )) && update_count=0top_delay=${top_delay:-0.5}
use_ps=${use_ps:-false}# check the directory of append-file(-a) mode, create if not exsit.
if [ -n "$append_file" ]; thenif [ -e "$append_file" ]; then[ -f "$append_file" ] || die "$append_file(specified by option -a, for storing run output files) exists but is not a file!"[ -w "$append_file" ] || die "file $append_file(specified by option -a, for storing run output files) exists but is not writable!"elseappend_file_dir="$(dirname "$append_file")"if [ -e "$append_file_dir" ]; then[ -d "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not a directory!"[ -w "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not writable!"elsemkdir -p "$append_file_dir" || die "fail to create directory $append_file_dir(specified by option -a, for storing run output files)!"fifi
fi# check store directory(-S) mode, create directory if not exsit.
if [ -n "$store_dir" ]; thenif [ -e "$store_dir" ]; then[ -d "$store_dir" ] || die "$store_dir(specified by option -S, for storing output files) exists but is not a directory!"[ -w "$store_dir" ] || die "directory $store_dir(specified by option -S, for storing output files) exists but is not writable!"elsemkdir -p "$store_dir" || die "fail to create directory $store_dir(specified by option -S, for storing output files)!"fi
fi################################################################################
# check the existence of jstack command
################################################################################if [ -n "$jstack_path" ]; then[ -f "$jstack_path" ] || die "$jstack_path is NOT found!"[ -x "$jstack_path" ] || die "$jstack_path is NOT executalbe!"
elif which jstack &> /dev/null; thenjstack_path="`which jstack`"
else[ -n "$JAVA_HOME" ] || die "jstack not found on PATH and No JAVA_HOME setting! Use -s option set jstack path manually."[ -f "$JAVA_HOME/bin/jstack" ] || die "jstack not found on PATH and \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) file does NOT exists! Use -s option set jstack path manually."[ -x "$JAVA_HOME/bin/jstack" ] || die "jstack not found on PATH and \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) is NOT executalbe! Use -s option set jstack path manually."jstack_path="$JAVA_HOME/bin/jstack"
fi################################################################################
# biz logic
################################################################################readonly run_timestamp="`date "+%Y-%m-%d_%H:%M:%S.%N"`"
readonly uuid="${PROG}_${run_timestamp}_${RANDOM}_$$"readonly tmp_store_dir="/tmp/${uuid}"
if [ -n "$store_dir" ]; thenreadonly store_file_prefix="$store_dir/${run_timestamp}_"
elsereadonly store_file_prefix="$tmp_store_dir/${run_timestamp}_"
fi
mkdir -p "$tmp_store_dir"cleanupWhenExit() {rm -rf "$tmp_store_dir" &> /dev/null
}
trap "cleanupWhenExit" EXITheadInfo() {colorEcho "0;34;42" ================================================================================echo "$(date "+%Y-%m-%d %H:%M:%S.%N") [$(( i + 1 ))/$update_count]: ${COMMAND_LINE[@]}"colorEcho "0;34;42" ================================================================================echo
}if [ -n "${pid}" ]; thenreadonly ps_process_select_options="-p $pid"
elsereadonly ps_process_select_options="-C java -C jsvc"
fi# output field: pid, thread id(lwp), pcpu, user
#   order by pcpu(percentage of cpu usage)
findBusyJavaThreadsByPs() {# 1. sort by %cpu by ps option `--sort -pcpu`# 2. use wide output(unlimited width) by ps option `-ww`#    avoid trunk user column to username_fo+ or $uid alikelocal -a ps_cmd_line=(ps $ps_process_select_options -wwLo pid,lwp,pcpu,user --sort -pcpu --no-headers)local -r ps_out="$("${ps_cmd_line[@]}")"if [ -n "$store_dir" ]; thenecho "$ps_out" | logAndCat "${ps_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_ps"fiecho "$ps_out" | head -n "${count}"
}# top with output field: thread id, %cpu
__top_threadId_cpu() {# 1. sort by %cpu by top option `-o %CPU`#    unfortunately, top version 3.2 does not support -o option(supports from top version 3.3+),#    use#       HOME="$tmp_store_dir" top -H -b -n 1#    combined#       sort#    instead of#       HOME="$tmp_store_dir" top -H -b -n 1 -o '%CPU'# 2. change HOME env var when run top,#    so as to prevent top command output format being change by .toprc user config file unexpectedly# 3. use option `-d 0.5`(update interval 0.5 second) and `-n 2`(update 2 times),#    and use second time update data to get cpu percentage of thread in 0.5 second interval# 4. top v3.3, there is 1 black line between 2 update;#    but top v3.2, there is 2 blank lines between 2 update!local -a top_cmd_line=(top -H -b -d $top_delay -n 2)local -r top_out=$(HOME="$tmp_store_dir" "${top_cmd_line[@]}")if [ -n "$store_dir" ]; thenecho "$top_out" | logAndCat "${top_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_top"fiecho "$top_out" |awk 'BEGIN { blockIndex = 0; currentLineHasText = 0; prevLineHasText = 0; } {currentLineHasText = ($0 != "")if (prevLineHasText && !currentLineHasText)blockIndex++    # from text line to empty line, increase block indexif (blockIndex == 3 && ($NF == "java" || $NF == "jsvc"))   # $NF(last field) is command field# only print 4th text block(blockIndex == 3), aka. process info of second top updateprint $1 " " $9     # $1 is thread id field, $9 is %cpu fieldprevLineHasText = currentLineHasText    # update prevLineHasText}' | sort -k2,2nr
}__complete_pid_user_by_ps() {# ps output field: pid, thread id(lwp), userlocal -a ps_cmd_line=(ps $ps_process_select_options -wwLo pid,lwp,user --no-headers)local -r ps_out="$("${ps_cmd_line[@]}")"if [ -n "$store_dir" ]; thenecho "$ps_out" | logAndCat "${ps_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_ps"filocal idx=0local -a linewhile IFS=" " read -a line ; do(( idx < count )) || breaklocal threadId="${line[0]}"local pcpu="${line[1]}"# output field: pid, threadId, pcpu, userlocal output_fields="$( echo "$ps_out" |awk -v "threadId=$threadId" -v "pcpu=$pcpu" '$2==threadId {printf "%s %s %s %s\n", $1, threadId, pcpu, $3; exit}' )"if [ -n "$output_fields" ]; then(( idx++ ))echo "$output_fields"fidone
}# output format is same as function findBusyJavaThreadsByPs
findBusyJavaThreadsByTop() {__top_threadId_cpu | __complete_pid_user_by_ps
}printStackOfThreads() {local -a linelocal idx=0while IFS=" " read -a line ; dolocal pid="${line[0]}"local threadId="${line[1]}"local threadId0x="0x`printf %x ${threadId}`"local pcpu="${line[2]}"local user="${line[3]}"(( idx++ ))local jstackFile="${store_file_prefix}$(( i + 1 ))_jstack_${pid}"[ -f "${jstackFile}" ] || {local -a jstack_cmd_line=( "$jstack_path" ${force} $mix_native_frames $more_lock_info ${pid} )if [ "${user}" == "${USER}" ]; then# run without sudo, when java process user is current userlogAndRun "${jstack_cmd_line[@]}" > ${jstackFile}elif [ $UID == 0 ]; then# if java process user is not current user, must run jstack with sudologAndRun sudo -u "${user}" "${jstack_cmd_line[@]}" > ${jstackFile}else# current user is not root user, so can not run with sudo; print error message and rerun suggestionredPrint "[$idx] Fail to jstack busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user})."redPrint "User of java process($user) is not current user($USER), need sudo to rerun:"yellowPrint "    sudo ${COMMAND_LINE[@]}"normalPrintcontinuefi || {redPrint "[$idx] Fail to jstack busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user})."normalPrintrm "${jstackFile}" &> /dev/nullcontinue}}bluePrint "[$idx] Busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user}):"if [ -n "$mix_native_frames" ]; thenlocal sed_script="/--------------- $threadId ---------------/,/^---------------/ {/--------------- $threadId ---------------/b # skip first separator line/^---------------/d # delete second separator linep}"elif [ -n "$force" ]; thenlocal sed_script="/^Thread ${threadId}:/,/^$/ {/^$/d; p # delete end separator line}"elselocal sed_script="/ nid=${threadId0x} /,/^$/ {/^$/d; p # delete end separator line}"fi{sed "$sed_script" -n ${jstackFile}echo} | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "${store_file_prefix}$PROG"}done
}################################################################################
# Main
################################################################################main() {local i# if update_count <= 0, infinite loop till user interrupted (eg: CTRL+C)for (( i = 0; update_count <= 0 || i < update_count; ++i )); do(( i > 0 )) && sleep "$update_delay"[ -n "$append_file" -o -n "$store_dir" ] && headInfo | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "${store_file_prefix}$PROG"} > /dev/null(( update_count != 1 )) && headInfoif $use_ps; thenfindBusyJavaThreadsByPselsefindBusyJavaThreadsByTopfi | printStackOfThreadsdone
}main