procdump分析

ProcDump是一款进程内存dump工具，可以针对内存，CPU，文件描述符等进行监控，使用命令如下：

# ./procdump -hProcDump v1.2 - Sysinternals process dump utility
Copyright (C) 2020 Microsoft Corporation. All rights reserved. Licensed under the MIT license.
Mark Russinovich, Mario Hewardt, John Salem, Javid Habibi
Monitors a process and writes a dump file when the process meets the
specified criteria.Usage: procdump [OPTIONS...] TARGETOPTIONS-h          Prints this help screen-C          Trigger core dump generation when CPU exceeds or equals specified value (0 to 100 * nCPU)-c          Trigger core dump generation when CPU is less than specified value (0 to 100 * nCPU)-M          Trigger core dump generation when memory commit exceeds or equals specified value (MB)-m          Trigger core dump generation when when memory commit is less than specified value (MB)-T          Trigger when thread count exceeds or equals specified value.-F          Trigger when file descriptor count exceeds or equals specified value.-G          Trigger when signal with the specified value (num) is sent (uses PTRACE and will affect performance of target process).-I          Polling frequency in milliseconds (default is 1000)-n          Number of core dumps to write before exiting (default is 1)-s          Consecutive seconds before dump is written (default is 10)-o          Path and/or filename prefix where the core dump is written to-d          Writes diagnostic logs to syslogTARGET must be exactly one of these:-p          pid of the process-w          Name of the process executable

本文来看一下ProcDump的实现原理。

1. Event

在Windows下面，事件是一个特别有用的内核对象，尤其是在当作通知使用的时候，例如一个进程需要等待系统初始化完成，那么就可以使用事件，当系统初始化完成之后就设置好事件；其实需要等待系统初始化完成的进程(线程)就等待该事件。

但是在Linux下面并没有事件通知这个东西，因此ProcMon就实现了事件，定义如下：

struct Event {pthread_mutex_t mutex;pthread_cond_t cond;bool bTriggered;bool bManualReset;char Name[MAX_EVENT_NAME];int nWaiters;
};

这里利用两个线程同步对象来实现：

1. pthread_mutex_t mutex; : 用来实现对Event的同步操作，例如触发状态。
2. pthread_cond_t cond; : 用来当作通知处理，例如当资源准备好的时候，需要通知其他等待线程的时候就使用条件变量。

针对事件，实现了如下函数：

struct Event *CreateEvent(bool IsManualReset, bool InitialState);
struct Event *CreateNamedEvent(bool IsManualReset, bool InitialState, char *Name);
void InitEvent(struct Event *Event, bool IsManualReset, bool InitialState);
void InitNamedEvent(struct Event *Event, bool IsManualReset, bool InitialState, char *Name);
void DestroyEvent(struct Event *Event);
bool SetEvent(struct Event *Event);
bool ResetEvent(struct Event *Event);

熟悉Windows的就会发现，这里跟Windows内核对象事件使用是一直，我们可以看一下其中几个主要的函数实现：

void InitNamedEvent(struct Event *Event, bool IsManualReset, bool InitialState, char *Name)
{static int unamedEventId = 0; // ID for logging purposespthread_mutex_init(&(Event->mutex), NULL);if(pthread_cond_init(&(Event->cond), NULL) != 0){Log(error, INTERNAL_ERROR);Trace("InitNamedEvent: failed pthread_cond_init.");exit(-1);}Event->bManualReset = IsManualReset;Event->bTriggered = InitialState;Event->nWaiters = 0;if (Name == NULL) {sprintf(Event->Name, "Unnamed Event %d", ++unamedEventId);} else if (strlen(Name) >= MAX_EVENT_NAME) {strncpy(Event->Name, Name, MAX_EVENT_NAME);Event->Name[MAX_EVENT_NAME - 1] = '\0'; // null terminate} else {strcpy(Event->Name, Name);}
}bool SetEvent(struct Event *Event)
{int success = 0;if ((success = pthread_mutex_lock(&(Event->mutex))) == 0) {     Event->bTriggered = true;Event->bManualReset ? // Are we a manual-reset?pthread_cond_broadcast(&(Event->cond)) :    // signal everyone!pthread_cond_signal(&(Event->cond));        // Signal first in line!pthread_mutex_unlock(&(Event->mutex));}else{Log(error, INTERNAL_ERROR);Trace("SetEvent: failed pthread_mutex_lock.");        exit(-1);}return (success == 0);
}bool ResetEvent(struct Event *Event)
{int success = 0;if ((success = pthread_mutex_lock(&(Event->mutex))) == 0) {Event->bTriggered = false;if(pthread_mutex_unlock(&(Event->mutex)) != 0){Log(error, INTERNAL_ERROR);Trace("ResetEvent: failed pthread_mutex_unlock.");        exit(-1);        }}else{Log(error, INTERNAL_ERROR);Trace("ResetEvent: failed pthread_mutex_lock.");        exit(-1);        }return (success == 0);
}

2. Handle

为了抽象对线程对象的处理，这里将线程对象（同步和通知）抽象成了Handle来进行处理，Handle的抽象代码如下：

enum EHandleType {EVENT,SEMAPHORE
};struct Handle {union {struct Event event;sem_t semaphore;};enum EHandleType type;
};

在Windows下面，对于内核对象的同步或者通知处理函数都是WaitForSingleObject，下面我们看一下这个函数的处理流程：

int WaitForSingleObject(struct Handle *Handle, int Milliseconds)
{struct timespec ts;int rc = 0;// Get current time and add wait timeif (Milliseconds != INFINITE_WAIT){ // We aren't waiting infinitelyclock_gettime(CLOCK_REALTIME, &ts);ts.tv_sec += Milliseconds / 1000;              // ms->sects.tv_nsec += (Milliseconds % 1000) * 1000000; // remaining ms->ns}switch (Handle->type) {case EVENT:if ((rc = pthread_mutex_lock(&(Handle->event.mutex))) == 0){Handle->event.nWaiters++;while (!Handle->event.bTriggered && rc == 0){rc = (Milliseconds == INFINITE_WAIT) ? // either waitpthread_cond_wait(&(Handle->event.cond), &(Handle->event.mutex)) :  // infinitelypthread_cond_timedwait(&(Handle->event.cond), &(Handle->event.mutex), &ts); // or till specified time passes}Handle->event.nWaiters--;// Check if we should resetif (Handle->event.nWaiters == 0 && !Handle->event.bManualReset){Handle->event.bTriggered = false;}pthread_mutex_unlock(&(Handle->event.mutex));}break;case SEMAPHORE:rc = (Milliseconds == INFINITE_WAIT) ? sem_wait(&(Handle->semaphore)) :sem_timedwait(&(Handle->semaphore), &ts);break;default:rc = -1;break;}return rc;
}

这个函数对于信号量的处理是比较简单的，直接调用sem_wait或者sem_timedwait函数，但是对于Event的处理就不一样了：

if ((rc = pthread_mutex_lock(&(Handle->event.mutex))) == 0)
{Handle->event.nWaiters++;while (!Handle->event.bTriggered && rc == 0){rc = (Milliseconds == INFINITE_WAIT) ? // either waitpthread_cond_wait(&(Handle->event.cond), &(Handle->event.mutex)) :  // infinitelypthread_cond_timedwait(&(Handle->event.cond), &(Handle->event.mutex), &ts); // or till specified time passes}Handle->event.nWaiters--;// Check if we should resetif (Handle->event.nWaiters == 0 && !Handle->event.bManualReset){Handle->event.bTriggered = false;}pthread_mutex_unlock(&(Handle->event.mutex));
}

这是一个典型的条件变量的使用方式，只是对于条件有一个是否手动重置的判断Handle->event.bManualReset。

3. 进程名

Windows下面获取进程名比较容易，因为Windows是一个比较开发的系统，一个进程可以操作另外一个进程；但是在Linux下面一个进程除了能给另外一个进程发送信号之外，不能做其他的任何操作了，那么如果知道一个进程的PID，怎么获取进程的名称呢？

在ProcDump中，是通过解析/proc/%d/cmdline来实现的，如下：

char * GetProcessName(pid_t pid){char procFilePath[32];char fileBuffer[MAX_CMDLINE_LEN];int charactersRead = 0;int	itr = 0;char * stringItr;char * processName;FILE * procFile;if(sprintf(procFilePath, "/proc/%d/cmdline", pid) < 0){return EMPTY_PROC_NAME;}procFile = fopen(procFilePath, "r");if(procFile != NULL){if(fgets(fileBuffer, MAX_CMDLINE_LEN, procFile) == NULL) {fclose(procFile);if(strlen(fileBuffer) == 0){Log(debug, "Empty cmdline.\n");}else{Log(debug, "Failed to read from %s.\n", procFilePath);}return EMPTY_PROC_NAME;}// close filefclose(procFile);}else{Log(debug, "Failed to open %s.\n", procFilePath);return EMPTY_PROC_NAME;}// Extract process namestringItr = fileBuffer;charactersRead  = strlen(fileBuffer);for(int i = 0; i <= charactersRead; i++){if(fileBuffer[i] == '\0'){itr = i - itr;if(strcmp(stringItr, "sudo") != 0){		// do we have the process name including filepath?processName = strrchr(stringItr, '/');	// does this process include a filepath?if(processName != NULL){return strdup(processName + 1);	// +1 to not include '/' character}else{return strdup(stringItr);}}else{stringItr += (itr+1); 	// +1 to move past '\0'}}}Log(debug, "Failed to extract process name from /proc/PID/cmdline");return EMPTY_PROC_NAME;
}

在这里有一个注意的地方就是需要去掉前面的sudo命令。

4. 进程枚举

同样Linux下面要想枚举系统的进程，例如ps等命令，方法就是枚举/proc/目录项，代码如下：

static int FilterForPid(const struct dirent *entry)
{return IsValidNumberArg(entry->d_name);
}bool WaitForProcessName(struct ProcDumpConfiguration *self)
{Log(info, "Waiting for process '%s' to launch...", self->ProcessName);while (true) {struct dirent ** nameList;bool moreThanOne = false;pid_t matchingPid = NO_PID;int numEntries = scandir("/proc/", &nameList, FilterForPid, alphasort);for (int i = 0; i < numEntries; i++) {pid_t procPid = atoi(nameList[i]->d_name);char *nameForPid = GetProcessName(procPid);if (strcmp(nameForPid, EMPTY_PROC_NAME) == 0) {continue;}if (strcmp(nameForPid, self->ProcessName) == 0) {if (matchingPid == NO_PID) {matchingPid = procPid;} else {Log(error, "More than one matching process found, exiting...");moreThanOne = true;free(nameForPid);break;}}free(nameForPid);}// Cleanupfor (int i = 0; i < numEntries; i++) {free(nameList[i]);}free(nameList);// Check for exactly one matchif (moreThanOne) {self->bTerminated = true;return false;} else if (matchingPid != NO_PID) {self->ProcessId = matchingPid;Log(info, "Found process with PID %d", matchingPid);return true;}}
}

通过IsValidNumberArg过滤条件来确定/proc/num的目录项当作是一个进程。

5. 进程状态获取

进程属性获取也是通过/proc/文件系统来完成的，读取的是/proc/[pid]/stat,代码如下：

bool GetProcessStat(pid_t pid, struct ProcessStat *proc) {char procFilePath[32];char fileBuffer[1024];char *token;char *savePtr = NULL;FILE *procFile = NULL;DIR* fddir = NULL;struct dirent* entry = NULL;// Get number of file descriptors in /proc/%d/fdinfo. This directory only contains sub directories for each file descriptor.if(sprintf(procFilePath, "/proc/%d/fdinfo", pid) < 0){return false;}fddir = opendir(procFilePath);if(fddir){proc->num_filedescriptors = 0;                 while ((entry = readdir(fddir)) != NULL){proc->num_filedescriptors++;}closedir(fddir);}else{Log(error, "Failed to open %s. Exiting...\n", procFilePath);return false; }proc->num_filedescriptors-=2;                   // Account for "." and ".."// Read /proc/[pid]/statif(sprintf(procFilePath, "/proc/%d/stat", pid) < 0){return false;}procFile = fopen(procFilePath, "r");if(procFile != NULL){if(fgets(fileBuffer, sizeof(fileBuffer), procFile) == NULL) {Log(error, "Failed to read from %s. Exiting...\n", procFilePath);fclose(procFile);return false;}// close file after reading this iteration of statsfclose(procFile);}else{Log(error, "Failed to open %s.\n", procFilePath);return false;}// (1) process IDproc->pid = (pid_t)atoi(fileBuffer);// (3) process stateif((savePtr = strrchr(fileBuffer, ')')) != NULL){savePtr += 2;     // iterate past ')' and ' ' in /proc/[pid]/statproc->state = strtok_r(savePtr, " ", &savePtr)[0];}// (4) parent process IDtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - Parent PID.");        return false;}proc->ppid = (pid_t)strtol(token, NULL, 10);// (5) process group IDtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - Process group ID.");        return false;}proc->pgrp = (gid_t)strtol(token, NULL, 10);// (6) session IDtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - Session ID.");        return false;}proc->session = (int)strtol(token, NULL, 10);// (7) controlling terminaltoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - Controlling terminal.");        return false;}proc->tty_nr = (int)strtol(token, NULL, 10);// (8) Foreground group ID of controlling terminaltoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - Foreground group ID.");        return false;}proc->tpgid = (gid_t)strtol(token, NULL, 10);// (9) Kernel flagstoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - Kernel flags.");        return false;}proc->flags = (unsigned int)strtoul(token, NULL, 10);// (10) minflttoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - Minflt.");        return false;}proc->minflt = strtoul(token, NULL, 10);// (11) cminflttoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - cminflt.");        return false;}proc->cminflt = strtoul(token, NULL, 10);// (12) majflttoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - majflt.");        return false;}proc->majflt = strtoul(token, NULL, 10);// (13) cmajflttoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - cmajflt.");        return false;}proc->cmajflt = strtoul(token, NULL, 10);// (14) utimetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - utime.");        return false;}proc->utime = strtoul(token, NULL, 10);// (15) stimetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - stime.");        return false;}proc->stime = strtoul(token, NULL, 10);// (16) cutimetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - cutime.");        return false;}proc->cutime = strtoul(token, NULL, 10);// (17) cstimetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - cstime.");        return false;}proc->cstime = strtoul(token, NULL, 10);// (18) prioritytoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - priority.");        return false;}proc->priority = strtol(token, NULL, 10);// (19) nicetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - nice.");        return false;}proc->nice = strtol(token, NULL, 10);// (20) num_threadstoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - num_threads.");       return false;}proc->num_threads = strtol(token, NULL, 10);// (21) itrealvaluetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - itrealvalue.");       return false;}proc->itrealvalue = strtol(token, NULL, 10);// (22) starttimetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - starttime.");       return false;}proc->starttime = strtoull(token, NULL, 10);// (23) vsizetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - vsize.");       return false;}proc->vsize = strtoul(token, NULL, 10);// (24) rsstoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - rss.");       return false;}proc->rss = strtol(token, NULL, 10);// (25) rsslimtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - rsslim.");       return false;}proc->rsslim = strtoul(token, NULL, 10);// (26) startcodetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - startcode.");       return false;}proc->startcode = strtoul(token, NULL, 10);// (27) endcodetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - endcode.");       return false;}proc->endcode = strtoul(token, NULL, 10);// (28) startstacktoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - startstack.");       return false;}proc->startstack = strtoul(token, NULL, 10);// (29) kstkesptoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - kstkesp.");       return false;}proc->kstkesp = strtoul(token, NULL, 10);// (30) kstkeiptoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - kstkeip.");       return false;}proc->kstkeip = strtoul(token, NULL, 10);// (31) signaltoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - signal.");       return false;}proc->signal = strtoul(token, NULL, 10);// (32) blockedtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - blocked.");       return false;}proc->blocked = strtoul(token, NULL, 10);// (33) sigignoretoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - sigignore.");       return false;}proc->sigignore = strtoul(token, NULL, 10);// (34) sigcatchtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - sigcatch.");       return false;}proc->sigcatch = strtoul(token, NULL, 10);// (35) wchantoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - wchan.");       return false;}proc->wchan = strtoul(token, NULL, 10);// (36) nswaptoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - nswap.");       return false;}proc->nswap = strtoul(token, NULL, 10);// (37) cnswaptoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - cnswap.");       return false;}proc->cnswap = strtoul(token, NULL, 10);// (38) exit_signaltoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - exit_signal.");       return false;}proc->exit_signal = (int)strtol(token, NULL, 10);// (39) processortoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - processor.");       return false;}proc->processor = (int)strtol(token, NULL, 10);// (40) rt_prioritytoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - rt_priority.");       return false;}proc->rt_priority = (unsigned int)strtoul(token, NULL, 10);// (41) policytoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - policy.");       return false;}proc->policy = (unsigned int)strtoul(token, NULL, 10);// (42) delayacct_blkio_tickstoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - delayacct_blkio_ticks.");       return false;}proc->delayacct_blkio_ticks = strtoull(token, NULL, 10);// (43) guest_timetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - guest_time.");       return false;}proc->guest_time = strtoul(token, NULL, 10);// (44) cguest_timetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - cguest_time.");       return false;}proc->cguest_time = strtol(token, NULL, 10);// (45) start_datatoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - start_data.");       return false;}proc->start_data = strtoul(token, NULL, 10);// (46) end_datatoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - end_data.");       return false;}proc->end_data = strtoul(token, NULL, 10);// (47) start_brktoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - start_brk.");       return false;}proc->end_data = strtoul(token, NULL, 10);// (48) arg_starttoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - arg_start.");return false;}proc->arg_start = strtoul(token, NULL, 10);// (49) arg_endtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - arg_end.");       return false;}proc->arg_end = strtoul(token, NULL, 10);// (50) env_starttoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - env_start.");return false;}proc->env_start = strtoul(token, NULL, 10);// (52) env_endtoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - env_end.");return false;}proc->env_end = strtoul(token, NULL, 10);// (53) exit_codetoken = strtok_r(NULL, " ", &savePtr);if(token == NULL){Trace("GetProcessStat: failed to get token from proc/[pid]/stat - exit_code.");return false;}proc->exit_code = (int)strtol(token, NULL, 10);return true;
}

对于/proc/[pid]/stat这个文件的格式如下：

Table 1-4: Contents of the stat files (as of 2.6.30-rc7)
..............................................................................Field          Contentpid           process idtcomm         filename of the executablestate         state (R is running, S is sleeping, D is sleeping in anuninterruptible wait, Z is zombie, T is traced or stopped)ppid          process id of the parent processpgrp          pgrp of the processsid           session idtty_nr        tty the process usestty_pgrp      pgrp of the ttyflags         task flagsmin_flt       number of minor faultscmin_flt      number of minor faults with child'smaj_flt       number of major faultscmaj_flt      number of major faults with child'sutime         user mode jiffiesstime         kernel mode jiffiescutime        user mode jiffies with child'scstime        kernel mode jiffies with child'spriority      priority levelnice          nice levelnum_threads   number of threadsit_real_value	(obsolete, always 0)start_time    time the process started after system bootvsize         virtual memory sizerss           resident set memory sizersslim        current limit in bytes on the rssstart_code    address above which program text can runend_code      address below which program text can runstart_stack   address of the start of the main process stackesp           current value of ESPeip           current value of EIPpending       bitmap of pending signalsblocked       bitmap of blocked signalssigign        bitmap of ignored signalssigcatch      bitmap of caught signals0		(place holder, used to be the wchan address, use /proc/PID/wchan instead)0             (place holder)0             (place holder)exit_signal   signal to send to parent thread on exittask_cpu      which CPU the task is scheduled onrt_priority   realtime prioritypolicy        scheduling policy (man sched_setscheduler)blkio_ticks   time spent waiting for block IOgtime         guest time of the task in jiffiescgtime        guest time of the task children in jiffiesstart_data    address above which program data+bss is placedend_data      address below which program data+bss is placedstart_brk     address above which program heap can be expanded with brk()arg_start     address above which program command line is placedarg_end       address below which program command line is placedenv_start     address above which program environment is placedenv_end       address below which program environment is placedexit_code     the thread's exit_code in the form reported by the waitpid system call
..............................................................................

6. 监控线程

ProcMon实现如下监控：

1. 提交的内存大小。
2. 线程数目。
3. 文件描述符数目。
4. CPU使用率（不过对于这里，CPU的计算是否有点欠妥）。

相关监控实现代码如下：

void *CommitMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */)
{Trace("CommitMonitoringThread: Starting Trigger Thread");struct ProcDumpConfiguration *config = (struct ProcDumpConfiguration *)thread_args;long pageSize_kb;unsigned long memUsage = 0;struct ProcessStat proc = {0};int rc = 0;struct CoreDumpWriter *writer = NewCoreDumpWriter(COMMIT, config); pageSize_kb = sysconf(_SC_PAGESIZE) >> 10; // convert bytes to kilobytes (2^10)if ((rc = WaitForQuitOrEvent(config, &config->evtStartMonitoring, INFINITE_WAIT)) == WAIT_OBJECT_0 + 1){while ((rc = WaitForQuit(config, config->PollingInterval)) == WAIT_TIMEOUT){if (GetProcessStat(config->ProcessId, &proc)){// Calc CommitmemUsage = (proc.rss * pageSize_kb) >> 10;    // get Resident Set SizememUsage += (proc.nswap * pageSize_kb) >> 10; // get Swap size// Commit Triggerif ((config->bMemoryTriggerBelowValue && (memUsage < config->MemoryThreshold)) ||(!config->bMemoryTriggerBelowValue && (memUsage >= config->MemoryThreshold))){Log(info, "Commit: %ld MB", memUsage);rc = WriteCoreDump(writer);if ((rc = WaitForQuit(config, config->ThresholdSeconds * 1000)) != WAIT_TIMEOUT){break;}}}else{Log(error, "An error occurred while parsing procfs\n");exit(-1);}            }}free(writer);Trace("CommitMonitoringThread: Exiting Trigger Thread");pthread_exit(NULL);
}void* ThreadCountMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */)
{Trace("ThreadCountMonitoringThread: Starting Thread Thread");struct ProcDumpConfiguration *config = (struct ProcDumpConfiguration *)thread_args;struct ProcessStat proc = {0};int rc = 0;struct CoreDumpWriter *writer = NewCoreDumpWriter(THREAD, config); if ((rc = WaitForQuitOrEvent(config, &config->evtStartMonitoring, INFINITE_WAIT)) == WAIT_OBJECT_0 + 1){while ((rc = WaitForQuit(config, config->PollingInterval)) == WAIT_TIMEOUT){if (GetProcessStat(config->ProcessId, &proc)){if (proc.num_threads >= config->ThreadThreshold){Log(info, "Threads: %ld", proc.num_threads);rc = WriteCoreDump(writer);if ((rc = WaitForQuit(config, config->ThresholdSeconds * 1000)) != WAIT_TIMEOUT){break;}}}else{Log(error, "An error occurred while parsing procfs\n");exit(-1);}            }}free(writer);Trace("ThreadCountMonitoringThread: Exiting Thread trigger Thread");pthread_exit(NULL);
}void* FileDescriptorCountMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */)
{Trace("FileDescriptorCountMonitoringThread: Starting Filedescriptor Thread");struct ProcDumpConfiguration *config = (struct ProcDumpConfiguration *)thread_args;struct ProcessStat proc = {0};int rc = 0;struct CoreDumpWriter *writer = NewCoreDumpWriter(FILEDESC, config); if ((rc = WaitForQuitOrEvent(config, &config->evtStartMonitoring, INFINITE_WAIT)) == WAIT_OBJECT_0 + 1){while ((rc = WaitForQuit(config, config->PollingInterval)) == WAIT_TIMEOUT){if (GetProcessStat(config->ProcessId, &proc)){if (proc.num_filedescriptors >= config->FileDescriptorThreshold){Log(info, "File descriptors: %ld", proc.num_filedescriptors);rc = WriteCoreDump(writer);if ((rc = WaitForQuit(config, config->ThresholdSeconds * 1000)) != WAIT_TIMEOUT){break;}}}else{Log(error, "An error occurred while parsing procfs\n");exit(-1);}            }}free(writer);Trace("FileDescriptorCountMonitoringThread: Exiting Filedescriptor trigger Thread");pthread_exit(NULL);
}
void *CpuMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */)
{Trace("CpuMonitoringThread: Starting Trigger Thread");struct ProcDumpConfiguration *config = (struct ProcDumpConfiguration *)thread_args;unsigned long totalTime = 0;unsigned long elapsedTime = 0;struct sysinfo sysInfo;int cpuUsage;struct CoreDumpWriter *writer = NewCoreDumpWriter(CPU, config);int rc = 0;struct ProcessStat proc = {0};if ((rc = WaitForQuitOrEvent(config, &config->evtStartMonitoring, INFINITE_WAIT)) == WAIT_OBJECT_0 + 1){while ((rc = WaitForQuit(config, config->PollingInterval)) == WAIT_TIMEOUT){sysinfo(&sysInfo);if (GetProcessStat(config->ProcessId, &proc)){// Calc CPUtotalTime = (unsigned long)((proc.utime + proc.stime) / HZ);elapsedTime = (unsigned long)(sysInfo.uptime - (long)(proc.starttime / HZ));cpuUsage = (int)(100 * ((double)totalTime / elapsedTime));// CPU Triggerif ((config->bCpuTriggerBelowValue && (cpuUsage < config->CpuThreshold)) ||(!config->bCpuTriggerBelowValue && (cpuUsage >= config->CpuThreshold))){Log(info, "CPU:\t%d%%", cpuUsage);rc = WriteCoreDump(writer);if ((rc = WaitForQuit(config, config->ThresholdSeconds * 1000)) != WAIT_TIMEOUT){break;}       }}else{Log(error, "An error occurred while parsing procfs\n");exit(-1);}}}free(writer);Trace("CpuTCpuMonitoringThread: Exiting Trigger Thread");pthread_exit(NULL);
}

当条件满足之后，调用WriteCoreDump来进行core文件的生成，这个函数主要调用gcore命令：

# gcore --help
usage:  gcore [-a] [-o filename] pid

整个ProcDump的核心就是在这几个工作线程下面来完成。