Thread Monitor

#*=============================================================================
#* Script Name:  Get-ThreadCPUpct.ps1
#*=============================================================================
#* Created:     01/01/2025
#* Author:      First Last
#*=============================================================================

###############################################################################
#
# DO NOT EDIT THE CODE DIRECTLY - Check it out of source control and use deploy.ps1
#
###############################################################################

<#
    .SYNOPSIS
    This script provides an automated way to monitor the stuck thread of a 
    Tomcat instance. If a thread is found to be stuck longer than 6 hours
    an email is sent, but only 1 email. Once the instance is restarted and
    the thread goes away, the script will adjust.    
#>

# Setup the log file name
$todayDate = Get-Date -Format "yyyyMMdd"
$Transcript = $PSScriptRoot + "\Log_${todayDate}.txt"
$fromEmail = 'monserver <DoNotReply.Monserver@domain.com>'
$toEmail = @(
    "alert.email.1@domain.com", 
    "alert.email.2@domain.com", 
    "alert.email.3@domain.com", 
    "alert.email.4@domain.com"
    )

# Start recording output to file
Start-Transcript -Path $Transcript -Append

# Define an array of remote server to query
$remoteServerNames = @(
    "SERVER-01", 
    "SERVER-02", 
    "SERVER-03",
    "SERVER-31", 
    "SERVER-32", 
    "SERVER-33",
    "SERVER-91", 
    "SERVER-92", 
    "SERVER-93"
    )

$ThreadAge = @{
    L="ThreadAge";
    E={[Timespan]::FromMilliseconds(
            $_.UserModeTime + $_.KernelModeTime
        ).ToString("dd\:hh\:mm\:ss")}
    }

$ThreadAgeD = @{
    L="ThreadAgeD";
    E={[Timespan]::FromMilliseconds(
            $_.UserModeTime + $_.KernelModeTime
        ).ToString("dd")}
    }

$ThreadAgeH = @{
    L="ThreadAgeH";
    E={[Timespan]::FromMilliseconds(
            $_.UserModeTime + $_.KernelModeTime
        ).ToString("hh")}
    }

$ThreadWaitReason = @{
    L = 'ThreadWaitReason'
    E = {
        # property is an array, so process all values
        $value = $_.ThreadWaitReason

        switch([int]$value)
        {
            0          {'Executive'}
            1          {'FreePage'}
            2          {'PageIn'}
            3          {'PoolAllocation'}
            4          {'ExecutionDelay'}
            5          {'FreePage'}
            6          {'PageIn'}
            7          {'Executive'}
            8          {'FreePage'}
            9          {'PageIn'}
            10         {'PoolAllocation'}
            11         {'ExecutionDelay'}
            12         {'FreePage'}
            13         {'PageIn'}
            14         {'EventPairHigh'}
            15         {'EventPairLow'}
            16         {'LPCReceive'}
            17         {'LPCReply'}
            18         {'VirtualMemory'}
            19         {'PageOut'}
            20         {'Rendezvous'}
            21         {'KeyedEvent'}
            22         {'Terminated'}
            23         {'ProcessInSwap'}
            24         {'CpuRateControl'}
            25         {'CalloutStack'}
            26         {'Kernel'}
            27         {'Resource'}
            28         {'PushLock'}
            29         {'Mutex'}
            30         {'QuantumEnd'}
            31         {'DispatchInt'}
            32         {'Preempted'}
            33         {'YieldExecution'}
            34         {'FastMutex'}
            35         {'GuardedMutex'}
            36         {'Rundown'}
            37         {'MaximumWaitReason'}
            default    {"$value"}
        }
    }  
}

$currSidFilePaths = @()
$serverWaitSecs = 5

# Connect to the remote server using WMI and retrieve thread information for Tomcat
$qryServer = @(
    "SELECT * "
    "FROM Win32_PerfFormattedData_PerfProc_Thread "
    "WHERE PercentProcessorTime > 95 AND Name LIKE '%Tomcat%'"
) -Join ""

foreach ($remoteServer in $remoteServerNames) {
    # Initialize retry count
    $retryCount = 1
    $errRetryCount = 1
    $maxRetries = 3
    $tCount = 0

    Write-Host "`nChecking threads on server: $remoteServer"
    Write-Host "=============================================================================="

    while (($errRetryCount -le $maxRetries) -and ($retryCount -le $maxRetries) -and ($tCount -lt 1)) {
        try {
            Write-Host "Try for threads: ${retryCount}"

            $threads = Get-WmiObject -Query $qryServer -ComputerName $remoteServer -ErrorAction Stop
            $errRetryCount = 1 # maxRetries errors per try

            $tCount = $threads | Measure-Object | Select-Object -ExpandProperty Count
            Write-Host "Thread count: ${tCount}"

            if ($tCount -lt 1) {             
                if ($retryCount -lt $maxRetries) {
                    Write-Host "Waiting ${serverWaitSecs}s and then retrying to double-check..."
                    Start-Sleep -Seconds $serverWaitSecs
                    Write-Host "Retrying WMI query..."
                } else {
                    Write-Host "Moving on..."
                }
                $retryCount++
            } else {
                 Write-Host "Found threads: $tCount"
            }

        } catch {
            $errRetryCount++
            Write-Host "Error try #${errRetryCount} failed: $_"
            Start-Sleep -Seconds $serverWaitSecs
        }
    } 

    if ($tCount -gt 0){
        # Iterate through each thread and get the owner process command line so we know which instance.
        foreach ($thread in $threads) {
            $processId = $thread.IDProcess
            $threadId = $thread.IDThread
            $pctCPU = $thread.PercentProcessorTime

            $qryProc = @(
                "SELECT CommandLine "
                "FROM Win32_Process "
                "WHERE ProcessId = $processId"
            ) -Join ""

            $qryThread = @(
                "SELECT * "
                "FROM Win32_Thread "
                "WHERE Handle = $threadId"
            ) -Join ""

            # Initialize retry count
            $retryCount = 1
            $maxRetries = 3

            :triesTime while ($retryCount -le $maxRetries) {
                try {
                    Write-Host "Try for cpu time: ${retryCount}"
                    $cpuTime = Get-WmiObject `
                        -Query $qryThread `
                        -ComputerName $remoteServer `
                        -ErrorAction Stop | `
                        Select-Object $ThreadAge, $ThreadAgeD, $ThreadAgeH, $ThreadWaitReason 
                    break triesTime
                } catch {
                    $retryCount++
                    Write-Error "Attempt $retryCount failed: $_"
                    Start-Sleep -Seconds $serverWaitSecs
                }
            }

            # Initialize retry count
            $retryCount = 1
            $maxRetries = 3

            :triesCmdL while ($retryCount -le $maxRetries) {
                try {
                    Write-Host "Try for command line: ${retryCount}"
                    $commandLine = (Get-WmiObject -Query $qryProc `
                                        -ComputerName $remoteServer `
                                        -ErrorAction Stop).CommandLine
                    break triesCmdL
                } catch {
                    $retryCount++
                    Write-Error "Attempt $retryCount failed: $_"
                    Start-Sleep -Seconds $serverWaitSecs
                }
            }

            $commandLine = $commandLine.Substring($commandLine.LastIndexOf("/") +1)

            $iFound = @(
                "Instance: ${commandLine} "
                "p-ID: ${processId} "
                "t-ID: ${threadId} "
                "t-CPU %: ${pctCPU} "
                "t-CPU time: $($cpuTime.ThreadAge) "
                "t-Reason: $($cpuTime.ThreadWaitReason) "
            ) -Join ""

            Write-Host $iFound

            # if the instance cputime is greater than 6 hours check if it's foundid file is present
            # we have a persistent, stuck thread)
            if (([int]$cpuTime.ThreadAgeD -gt 0) -or ([int]$cpuTime.ThreadAgeH -ge 6)) 
            {
                $sidserver = $remoteServer.Substring($remoteServer.LastIndexOf("-") +1) #03
                $sidinst = $commandLine.Substring($commandLine.Length - 2) #07
                $sid = $sidserver + $sidinst + $processId + $threadId #sid03078285964 
                $sidFilePath = $PSScriptRoot + "\sid${sid}"

                $iMessage = @(
                    "If an opportunity arises you can restart the intance in off hours."
                    "Make sure to Log this occurance."
                    "Take a look at today's log to determine the start time."
                    "Log is located here:"
                    "  MONSERVER-02 - ${Transcript}"
                ) -Join "`n"

                if (-Not (Test-Path -Path $sidFilePath)) {
                    # Send the message using the internal relay
                    Send-MailMessage `
                        -From $fromEmail `
                        -To $toEmail `
                        -Subject "_Alert - APP - Thread Monitor" `
                        -Body "${remoteServer} - ${iFound}`n`n${iMessage}" `
                        -Priority High `
                        -SmtpServer 'relay.domain.com'

                    New-Item -Path $sidFilePath -ItemType File | Out-Null
                }

                $currSidFilePaths += $sidFilePath
            }
        }
    }
}

# At the end of the script, get all sid files in the script directory.
$allSidFiles = Get-ChildItem -Path $PSScriptRoot -File -Filter "sid*.*"

# Clear out sid files that are no longer needed.
foreach ($sidFile in $allSidFiles) {

    if (-Not ($currSidFilePaths -contains $sidFile.FullName)) {
        # File is not in the array, delete it
        Remove-Item -Path $sidFile.FullName -Force
        Write-Output "Deleted file: $($sidFile.FullName)"
    }
}

Stop-Transcript

# Get the files in the directory
$files = Get-ChildItem -Path $PSScriptRoot -File -Filter "*.txt"

# Count the number of files in the directory
$fileCount = $files.Count

# Check if the number of files is greater than 90
if ($fileCount -gt 90) {
    # Get the oldest file in the directory
    $oldestFile = $files | Sort-Object LastWriteTime | Select-Object -First 1

    # Delete the oldest file
    Remove-Item $oldestFile.FullName
}