PowerShell script to delete cache on SPOC, SPOC Group

<#
.SYNOPSIS
   YSoft SAFEQ 6 SPOC group restart and cache deletion script
 
.DESCRIPTION
   Script helps to reinitialize the SPOC / SPOC group by SPOC cache deletion.
   - It automates the steps described in the documentation.
   - It can delete Job Service cache when configured so.
   - The cache (SPOC, JS), the old log files (SPOC, Terminal Server) can be found in the original location with suffix _backup_<datetime> . You can delete them if you do not need them for later analysis.
    
   Run the script as follows:
   - Make sure YSoft SafeQ Management Service is up and running on all management servers.
   - Run the script on all members of a single SPOC group.
   - Follow on-scren instructions to stop SAFEQ services on all SPOC servers at once.
   - Once the script displays "To restart whole SPOC Group, ..." on all the servers:
   -- follow on-screen instructions on one of the servers. Script will delete caches and start SAFEQ services.
   -- once the initialization of first server finishes, you may continue with initialization of another server (one by one).
   - Afterwards it is recommended to check that all services are really started.
    
   If the script fails on single node when restarting whole SPOC group, re-run the script on all nodes again.
 
   Script can be also used for a cache deletion on a standalone SPOC.
 
.EXAMPLE
   Run the PowerShell as Administrator, then launch the script in it.
 
.NOTES
  Version:        1.16
  Last Modified:  15/July/2024
#>
 
#-----------------------------------------------------------[Parameters]-----------------------------------------------------------
 
# Start TS service (and other services) automatically when SPOC finishes cache recovery ($true) or wait for command from administrator ($false)
$tsautostart = $true
 
# Delete also Job Service cache ($true) or leave it intact ($false), default value is $false and we suggest to keep it unless Y Soft specifically suggests otherwise
$JScachecleanup = $false

# Move the old cache (SPOC, JS) and the old log files (SPOC, Terminal Server) to the backup location ($true) or just delete them ($false)
$keepbackup = $true

#-----------------------------------------------------------[Execution]------------------------------------------------------------
 
#Admin rights check
If (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole(([System.Security.Principal.SecurityIdentifier]'S-1-5-32-544'))) {
    Write-Warning 'Administrative rights are missing. Please re-run the script as an Administrator.'
    Read-Host 'Press any key to exit the script'
    exit
}
 
#Check path and set it manually if required
$pm_sqdir = Get-ChildItem -Path HKLM:\SYSTEM\CurrentControlSet\Services | ? {(($_ | Get-ItemProperty).PSChildName) -eq 'YSoftSQ-SPOC'}
if (!$pm_sqdir) {
    Read-Host 'Spooler Controller service not found. Terminating.'
    throw
}
$pm_sqdir = ($pm_sqdir | Get-ItemProperty).ImagePath.Split()[0].Trim('`"') -replace ('\\?bin\\wrapper.exe?','')
Write-Output ((Get-Date).ToString("HH:mm:ss") + " Spooler Controller path set to: $pm_sqdir")

if ($JScachecleanup -eq $true){
    $pm_JSdir = Get-ChildItem -Path HKLM:\SYSTEM\CurrentControlSet\Services | ? {(($_ | Get-ItemProperty).PSChildName) -eq 'YSoftSQ-JOB-SERVICE'}
    if (!$pm_JSdir) {
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " Job Service not found, JS cache cleanup will not be made")
        $JScachecleanup = $false
    } else {
        $pm_JSdir = ($pm_JSdir | Get-ItemProperty).ImagePath.Split()[0].Trim('`"') -replace ('\\YSoft.JobService.Host.exe?','')
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " Job Service path set to: $pm_JSdir")
    }
}
 
#Function to get list of the services to start/stop/restart during the cache recovery process
function listservice ($phase) {
    if ($phase -eq 'one')
    {
        Get-Service *YSoft* -Exclude YSoftPGSQL, YYSoftPGSQL, YSoftSQ-Management, YSoftSQ-LDAP, YSoftIms
    }
    elseif ($phase -eq 'two')
    {
        Get-Service *YSoft* -Exclude YSoftPGSQL, YYSoftPGSQL, YSoftSQ-Management, YSoftSQ-LDAP, YSoftIms, YSoft*TS*, YSoft*TerminalServer, YSoftSQ-SPOC, YSoftSQ-SPOCGS, YSoftSQ-JSDL, YSoftSQ-SPOOLER, YSoftSQ-JOB-SERVICE | Where-Object {$_.StartType -ne 'Manual' -and $_.StartType -ne 'Disabled'}
    }
    else
    {
        Throw '"listservice" was called without any parameter, the script likely contains an error. Try to download the script again.'
    }
 
}
 
#Stop YSoft SAFEQ services
Read-Host ((Get-Date).ToString("HH:mm:ss") + " Press Enter to stop YSoft SAFEQ services")
Write-Output ((Get-Date).ToString("HH:mm:ss") + " Stopping services (if a service takes more than 5 minutes to stop, kill the associated process via Task Manager; see service properties for process name)")

listservice -phase 'one' | Stop-Service -Force -ErrorAction SilentlyContinue

$proc = Get-Process '*MigService*'
if ($proc.Count -ge 1)
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " YSoftSQ-MIG service cannot be stopped, killing it by force")
    $proc | Select-Object Id, ProcessName, Path | Stop-Process -Force
    Start-Sleep -Seconds 4
}

# L4C-3786 workaround, can be removed once associated defect is fixed
$proc = Get-Process '*TerminalServer*'
if ($proc.Count -ge 1)
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " YSoftSQ-TS service cannot be stopped, killing it by force")
    $proc | Select-Object Id, ProcessName, Path | Stop-Process -Force
    Start-Sleep -Seconds 4
}

if (((( listservice -phase 'one' | Where-Object {$_.Status -ne 'Stopped'}) | Measure-Object ).Count) -gt 0)
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " Following services are still running: " + $((listservice -phase 'one' | Where-Object {$_.Status -ne 'Stopped'}).DisplayName -join ', '))
    Throw "Failed to stop services listed above, stop them manually or kill the process associated with them via Task Manager (see service properties for process name). Then start the script on this server again."
}
 
Write-Output ((Get-Date).ToString("HH:mm:ss") + " Services stopped")
Read-Host ((Get-Date).ToString("HH:mm:ss") + " To restart whole SPOC Group, stop services on all members of the group now. When done press any key to continue on one of the nodes")
 
#Backup directories

if ($JScachecleanup -eq $true){
    $pm_foldertorename = "$pm_sqdir\logs","$pm_sqdir\terminalserver\logs","$pm_sqdir\SpoolCache","$pm_JSdir\distServer\data\jobService\index"
}
else{
    $pm_foldertorename = "$pm_sqdir\logs","$pm_sqdir\terminalserver\logs","$pm_sqdir\SpoolCache"
}

if ($keepbackup -eq $false){
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " Deleting cache and log directory")
    foreach ($pm_folder in $pm_foldertorename) {
        Remove-Item -Path $pm_folder -Recurse -ErrorAction SilentlyContinue
        if (-not (Test-Path $pm_folder)) {" Deleted: $pm_folder"}
    }
}
else {
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " Renaming cache and log directory")
    $timestamp = (Get-Date).ToString("yyyy-MM-dd_HH-mm-ss")
    foreach ($pm_folder in $pm_foldertorename) {
        $pm_foldernewname = $pm_folder+"_backup_"+$timestamp
        Rename-Item -Path $pm_folder -NewName $pm_foldernewname -ErrorAction SilentlyContinue
        if (Test-Path $pm_foldernewname) {" Renamed: $pm_folder to $($pm_foldernewname.Split('\')[-1])"}
    }
}
 
if (Test-Path -Path $pm_sqdir\SpoolCache)
{
    Throw ((Get-Date).ToString("HH:mm:ss") + " Terminating as deleting SPOC cache has failed. Make sure all the services are stopped, then use the script again.")
}
elseif ($JScachecleanup -eq $true -and (Test-Path -Path $pm_JSdir\distServer\data\jobService\index))
{
    Throw ((Get-Date).ToString("HH:mm:ss") + " Terminating as deleting JS cache has failed. Make sure all the services are stopped, then use the script again.")
}
else
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " Renaming cache and log directory finished")
}

#Start YSoftSQ-JSDL service if exists
if (Get-Service *YSoftSQ-JSDL*)
{
    try
    {
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " Starting YSoft SafeQ Job Service Distributed Layer")
        # Restart-Service is better than Start-Service, especially in situation when script was launched shortly after OS reboot and services are set for delayed start
        Get-Service *YSoftSQ-JSDL* | Restart-Service -ErrorAction Stop
    }
    catch
    {
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " Starting YSoft SafeQ Job Service Distributed Layer service has failed. Please try it again manually once the rest of steps is finished.")
    }
    finally
    {
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " JSDL service started")
    }

}

#Start SPOC service and wait for initialization
Write-Output ((Get-Date).ToString("HH:mm:ss") + " Starting YSoft SAFEQ SPOC service")
if ((Get-Service YSoftSQ-SPOC).Status -eq 'Stopped')
{
    try
    {
        Start-Service YSoftSQ-SPOC -ErrorAction Stop
    }
    catch
    {
        Throw "Starting YSoftSQ-SPOC service has failed, terminating"
    }
    finally {}
}
else
{
    Throw "YSoftSQ-SPOC is already running, this is unexpected. Start the script again."
}
 
try
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " Waiting for $pm_sqdir\conf\remoteConfImg.xml to be created")
    do {Start-Sleep 2 } until (Test-Path $pm_sqdir\conf\remoteConfImg.xml)
    [xml]$remoteConf= get-content $pm_sqdir\conf\remoteConfImg.xml
    $pm_orsFailoverLockManager = $remoteConf.configuration.property | Where-Object {$_.key -eq 'orsFailoverLockManager'}
 
    if ($pm_orsFailoverLockManager.Value -eq 'true')
    {
        $pm_lookupstring = 'Download\sentities\sfinished\sfrom\sother\sORSes\sin\sNRG'
    }
 
    elseif ($pm_orsFailoverLockManager.Value -eq 'false')
    {
        $pm_lookupstring = 'End\sof\sprocessing\sof\sGetNewJobsByUsersResponseMessage'
    }
}
catch
{
    Throw "Error occurred while trying to verify configuration in remoteConfImg.xml, terminating"
}
finally
{
    if (-Not($pm_lookupstring))
    {
        Throw "Property orsFailoverLockManager not found in remoteConfImg.xml, terminating"
    }
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " SPOC initialization in progress, please wait")
}
 
do
{
    Start-Sleep -Seconds 10
    $spocLog = Get-Content -Encoding String -Path $pm_sqdir\logs\spoc.log
    [array]::Reverse($spocLog)
    $spocLogMatch = $spocLog | Where-Object { $_ -match $pm_lookupstring } | Select-Object -First 1
}
until (($spocLogMatch | Measure-Object).Count -gt 0)
Write-Output ((Get-Date).ToString("HH:mm:ss") + " SPOC initialization finished")
Write-Output ((Get-Date).ToString("HH:mm:ss") + " If you were restarting the whole SPOC group, you may continue with startup of the next SPOC group member.")
 
#Start TS service and wait for initialization
if ($tsautostart -eq $false)
{
    Read-Host ((Get-Date).ToString("HH:mm:ss") + " Press Enter to start YSoft SAFEQ Terminal Server and other remaining services")
}

try
{
    if (Get-Service *YSoftSQ-SPOOLER*) {
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " Starting YSoft SAFEQ Terminal Server and YSoft SafeQ Client v3 related services") 
    } else { 
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " Starting YSoft SAFEQ Terminal Server service") 
    }

    # L4C-3786 workaround, can be removed once associated defect is fixed
    $proc = Get-Process '*TerminalServer*'
    if ($proc.Count -ge 1)
    {
        Write-Output ((Get-Date).ToString("HH:mm:ss") + " YSoft SAFEQ Terminal Server service is already running, that is unexpected. Initiating forced termination at first to ensure a smooth startup.")
        $proc | Select-Object Id, ProcessName, Path | Stop-Process -Force
        Start-Sleep -Seconds 4

        $pm_foldertorename = "$pm_sqdir\terminalserver\logs"

        if ($keepbackup -eq $false){
            Write-Output ((Get-Date).ToString("HH:mm:ss") + " Deleting TS log directory")
            foreach ($pm_folder in $pm_foldertorename) {
                Remove-Item -Path $pm_folder -Recurse -ErrorAction SilentlyContinue
                if (-not (Test-Path $pm_folder)) {" Deleted: $pm_folder"}
            }
        }
        else {
            Write-Output ((Get-Date).ToString("HH:mm:ss") + " Renaming TS log directory")
            $timestamp = (Get-Date).ToString("yyyy-MM-dd_HH-mm-ss")
            foreach ($pm_folder in $pm_foldertorename) {
                $pm_foldernewname = $pm_folder+"_backup_"+$timestamp
                Rename-Item -Path $pm_folder -NewName $pm_foldernewname -ErrorAction SilentlyContinue
                if (Test-Path $pm_foldernewname) {" Renamed: $pm_folder to $($pm_foldernewname.Split('\')[-1])"}
            }
        }
    }


    # Restart-Service is better than Start-Service, especially in situation when script was launched shortly after OS reboot and services are set for delayed start, JS must start before SPOOLER in SQ6 Build 79 and below (SBT-4477)
    Get-Service YSoft*TS*, YSoft*TerminalServer | Restart-Service -ErrorAction Stop
    Get-Service *YSoftSQ-JOB-SERVICE*, *YSoftSQ-SPOOLER* | Restart-Service -ErrorAction Stop
}
catch
{
    if (Get-Service *YSoftSQ-SPOOLER*) {
        Throw ((Get-Date).ToString("HH:mm:ss") + " Starting YSoft SAFEQ Terminal Server, YSoft SafeQ Spooler or YSoft SafeQ Job Service has failed, terminating. Please try to start it manually and then start all the remaining services.")
    } else { 
        Throw ((Get-Date).ToString("HH:mm:ss") + " Starting YSoft SAFEQ Terminal Server service has failed, terminating. Please try to start it manually and then start all the remaining services.") 
    }
}
finally
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " TS initialization in progress, please wait")
}

do
{
    Start-Sleep -Seconds 10
    $tslog = Get-Content -Encoding String -Path $pm_sqdir\terminalserver\logs\terminalserver.log
    [array]::Reverse($tslog)
    $tslogMatch = $tslog | Where-Object { $_ -match 'TS\sfully\sstarted' } | Select-Object -First 1
}
until (($tslogMatch | Measure-Object).Count -gt 0)
Write-Output ((Get-Date).ToString("HH:mm:ss") + " TS initialization finished")
        
 
#Start remaining services and wait for initialization
Write-Output ((Get-Date).ToString("HH:mm:ss") + " Starting any remaining YSoft SAFEQ services that were also stopped")
try
{
    # Restart-Service is better than Start-Service, especially in situation when script was launched shortly after OS reboot and services are set for delayed start
    listservice -phase 'two' | Restart-Service -ErrorAction Stop 
}
catch
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " Startup of some additional YSoft SAFEQ services has failed. Make sure to start all remaining YSoft SAFEQ services manually.")
}
finally
{
    Write-Output ((Get-Date).ToString("HH:mm:ss") + " FINISHED - services are fully initialized" )
}