Add resilient upload queue and client health

config/ocsentinel-client.dev.example.json

@@ -6,6 +6,7 @@
   "n8nWebhookUrl": "http://172.16.41.197:5678/webhook/ocsentinel-ingest",
   "deviceIdentifierMode": "machineName",
   "uploadTimeoutSeconds": 30,
+  "uploadQueueMaxReports": 100,
   "enableVulnerabilityCorrelation": true,
   "vulnerabilityCsvPath": "",
   "secretReference": "device-default"

config/ocsentinel-client.example.json

@@ -6,6 +6,7 @@
   "n8nWebhookUrl": "https://n8n.example.com/webhook/ocsentinel-ingest",
   "deviceIdentifierMode": "machineName",
   "uploadTimeoutSeconds": 30,
+  "uploadQueueMaxReports": 100,
   "enableVulnerabilityCorrelation": true,
   "vulnerabilityCsvPath": "",
   "secretReference": "device-default"

docs/ocsentinel-deployment.md

@@ -46,6 +46,24 @@ automation read access. Set it to `true` for a device to begin the five-minute
 burst scans; clear it to stop them. The normal daily scan continues regardless
 of the checkbox.
 
+## Upload Reliability And Client Health
+
+If the upload endpoint is temporarily unavailable, the client stores up to 100
+signed report payloads locally under `C:\ProgramData\OCSentinel\upload-queue`.
+The next scheduled run sends queued payloads before its new report. The local
+health state is stored under `C:\ProgramData\OCSentinel\state`.
+
+Create these additional device custom fields in NinjaOne and allow automation
+write access:
+
+| Field name | Type | Purpose |
+| --- | --- | --- |
+| `ocsentineluploadstatus` | Text | `ok`, `queued`, or `unknown` upload state |
+| `ocsentinelqueuedreports` | Integer | Reports waiting for delivery |
+| `ocsentinellastuploadutc` | Date/Time | Last successful upload time |
+| `ocsentinellasterror` | Text | Last upload error, if any |
+| `ocsentinelclientversion` | Text | Installed client version |
+
 ## NinjaOne Tasks
 
 Create a PowerShell script in NinjaOne named `OCSentinel - Installieren oder aktualisieren`.

installer/runtime-run-ocsentinel-monitor.ps1

@@ -124,6 +124,16 @@ function Publish-NinjaCustomFields {
         }
     }
 
+    $uploadStatus = [string]$Report.Runtime.UploadStatus
+    if ([string]::IsNullOrWhiteSpace($uploadStatus)) { $uploadStatus = "unknown" }
+    $queuedReports = [int]$Report.Runtime.QueuedReportCount
+    $lastUploadUtc = ""
+    if ($Report.Runtime.LastSuccessfulUploadUtc) {
+        try { $lastUploadUtc = ([DateTimeOffset]$Report.Runtime.LastSuccessfulUploadUtc).ToUniversalTime().ToString("o") } catch { $lastUploadUtc = [string]$Report.Runtime.LastSuccessfulUploadUtc }
+    }
+    $lastUploadError = [string]$Report.Runtime.LastUploadError
+    if ($lastUploadError.Length -gt 900) { $lastUploadError = $lastUploadError.Substring(0, 900) }
+
     $fieldValues = @(
         [pscustomobject]@{ Name = "ocsentinelstatus"; Type = "Text"; Value = [string]$Report.AlertState }
         [pscustomobject]@{ Name = "ocsentinelreason"; Type = "Text"; Value = $Reason }
@@ -135,6 +145,11 @@ function Publish-NinjaCustomFields {
         [pscustomobject]@{ Name = "ocsentinelmode"; Type = "Text"; Value = $Mode }
         [pscustomobject]@{ Name = "ocsentineltriggered"; Type = "Checkbox"; Value = $Triggered }
         [pscustomobject]@{ Name = "ocsentinellastscanutc"; Type = "DateTime"; Value = $generatedAtUtc }
+        [pscustomobject]@{ Name = "ocsentineluploadstatus"; Type = "Text"; Value = $uploadStatus }
+        [pscustomobject]@{ Name = "ocsentinelqueuedreports"; Type = "Integer"; Value = $queuedReports }
+        [pscustomobject]@{ Name = "ocsentinellastuploadutc"; Type = "DateTime"; Value = $lastUploadUtc }
+        [pscustomobject]@{ Name = "ocsentinellasterror"; Type = "Text"; Value = $lastUploadError }
+        [pscustomobject]@{ Name = "ocsentinelclientversion"; Type = "Text"; Value = [string]$Report.ClientVersion }
     )
 
     $updated = 0
@@ -190,6 +205,8 @@ $events = [int]$report.TotalEvents
 $uniqueIps = [int]$report.UniqueIpCount
 $criticalCves = [int]$report.VulnerabilityCorrelation.CriticalCount
 $totalCves = [int]$report.VulnerabilityCorrelation.TotalCount
+$uploadStatus = [string]$report.Runtime.UploadStatus
+$queuedReports = [int]$report.Runtime.QueuedReportCount
 
 $monitorTriggered = $false
 $monitorReason = ""
@@ -225,6 +242,8 @@ Write-Host "Events: $events"
 Write-Host "Unique IPs: $uniqueIps"
 Write-Host "Critical/High CVEs: $criticalCves"
 Write-Host "Total CVEs: $totalCves"
+Write-Host "Upload status: $uploadStatus"
+Write-Host "Queued reports: $queuedReports"
 Write-Host "Report: $outputFullPath"
 Write-Host "Runner exit code: $runnerExitCode"
 

scripts/run-ocsentinel-monitor.ps1

@@ -124,6 +124,16 @@ function Publish-NinjaCustomFields {
         }
     }
 
+    $uploadStatus = [string]$Report.Runtime.UploadStatus
+    if ([string]::IsNullOrWhiteSpace($uploadStatus)) { $uploadStatus = "unknown" }
+    $queuedReports = [int]$Report.Runtime.QueuedReportCount
+    $lastUploadUtc = ""
+    if ($Report.Runtime.LastSuccessfulUploadUtc) {
+        try { $lastUploadUtc = ([DateTimeOffset]$Report.Runtime.LastSuccessfulUploadUtc).ToUniversalTime().ToString("o") } catch { $lastUploadUtc = [string]$Report.Runtime.LastSuccessfulUploadUtc }
+    }
+    $lastUploadError = [string]$Report.Runtime.LastUploadError
+    if ($lastUploadError.Length -gt 900) { $lastUploadError = $lastUploadError.Substring(0, 900) }
+
     $fieldValues = @(
         [pscustomobject]@{ Name = "ocsentinelstatus"; Type = "Text"; Value = [string]$Report.AlertState }
         [pscustomobject]@{ Name = "ocsentinelreason"; Type = "Text"; Value = $Reason }
@@ -135,6 +145,11 @@ function Publish-NinjaCustomFields {
         [pscustomobject]@{ Name = "ocsentinelmode"; Type = "Text"; Value = $Mode }
         [pscustomobject]@{ Name = "ocsentineltriggered"; Type = "Checkbox"; Value = $Triggered }
         [pscustomobject]@{ Name = "ocsentinellastscanutc"; Type = "DateTime"; Value = $generatedAtUtc }
+        [pscustomobject]@{ Name = "ocsentineluploadstatus"; Type = "Text"; Value = $uploadStatus }
+        [pscustomobject]@{ Name = "ocsentinelqueuedreports"; Type = "Integer"; Value = $queuedReports }
+        [pscustomobject]@{ Name = "ocsentinellastuploadutc"; Type = "DateTime"; Value = $lastUploadUtc }
+        [pscustomobject]@{ Name = "ocsentinellasterror"; Type = "Text"; Value = $lastUploadError }
+        [pscustomobject]@{ Name = "ocsentinelclientversion"; Type = "Text"; Value = [string]$Report.ClientVersion }
     )
 
     $updated = 0
@@ -190,6 +205,8 @@ $events = [int]$report.TotalEvents
 $uniqueIps = [int]$report.UniqueIpCount
 $criticalCves = [int]$report.VulnerabilityCorrelation.CriticalCount
 $totalCves = [int]$report.VulnerabilityCorrelation.TotalCount
+$uploadStatus = [string]$report.Runtime.UploadStatus
+$queuedReports = [int]$report.Runtime.QueuedReportCount
 
 $monitorTriggered = $false
 $monitorReason = ""
@@ -225,6 +242,8 @@ Write-Host "Events: $events"
 Write-Host "Unique IPs: $uniqueIps"
 Write-Host "Critical/High CVEs: $criticalCves"
 Write-Host "Total CVEs: $totalCves"
+Write-Host "Upload status: $uploadStatus"
+Write-Host "Queued reports: $queuedReports"
 Write-Host "Report: $outputFullPath"
 Write-Host "Runner exit code: $runnerExitCode"
 

src/OCSentinelCli/Commands/UploadCommand.cs

@@ -1,4 +1,5 @@
 using OCSentinelCli.Configuration;
+using OCSentinelCli.Models;
 using System.Text.Json;
 using OCSentinelCli.Security;
 using OCSentinelCli.Transport;
@@ -56,30 +57,52 @@ internal static class UploadCommand
 
         string reportFullPath = Path.GetFullPath(reportPath);
         string json = File.ReadAllText(reportFullPath);
+        string secret = ProtectedSecretStore.LoadSecret(resolvedSecretPath);
+        var client = new N8nUploadClient();
+        var queue = new UploadQueue(config.UploadQueueMaxReports);
+        UploadHealth health = queue.LoadHealth();
+        DateTimeOffset attemptTime = DateTimeOffset.UtcNow;
         ScanResult? parsedReport = JsonSerializer.Deserialize<ScanResult>(json, JsonOptions.Default);
+
         if (parsedReport is not null)
         {
             parsedReport = parsedReport with
             {
                 Runtime = parsedReport.Runtime with
                 {
-                    UploadAttempted = true
+                    UploadAttempted = true,
+                    UploadStatus = "attempting",
+                    QueuedReportCount = health.QueuedReportCount,
+                    LastSuccessfulUploadUtc = health.LastSuccessfulUploadUtc
                 }
             };
-
             json = JsonSerializer.Serialize(parsedReport, JsonOptions.Default);
             File.WriteAllText(reportFullPath, json);
         }
 
-        string secret = ProtectedSecretStore.LoadSecret(resolvedSecretPath);
+        UploadResult? deferredFailure = queue.Drain(
-        var client = new N8nUploadClient();
+            client,
-        var result = client.UploadJson(config.N8nWebhookUrl, Environment.MachineName, BuildMetadata.Version, json, secret, config.UploadTimeoutSeconds);
+            config.N8nWebhookUrl,
+            Environment.MachineName,
+            BuildMetadata.Version,
+            secret,
+            config.UploadTimeoutSeconds);
 
-        if (!result.Success)
+        if (deferredFailure is null)
         {
-            Console.Error.WriteLine($"Upload failed ({result.StatusCode}): {result.Message}");
+            var result = client.UploadJson(config.N8nWebhookUrl, Environment.MachineName, BuildMetadata.Version, json, secret, config.UploadTimeoutSeconds);
-            return 1;
+            if (result.Success)
-        }
+            {
+                UploadHealth successHealth = new()
+                {
+                    LastUploadAttemptUtc = attemptTime,
+                    LastSuccessfulUploadUtc = DateTimeOffset.UtcNow,
+                    LastUploadStatus = "ok",
+                    LastUploadError = string.Empty,
+                    QueuedReportCount = queue.GetQueueDepth()
+                };
+                queue.SaveHealth(successHealth);
+                WriteReportRuntime(reportFullPath, parsedReport, successHealth, true);
 
                 Console.WriteLine($"Upload succeeded ({result.StatusCode})");
                 Console.WriteLine($"Nonce: {result.Nonce}");
@@ -87,6 +110,50 @@ internal static class UploadCommand
                 return 0;
             }
 
+            deferredFailure = result;
+        }
+
+        UploadResult failure = deferredFailure ?? throw new InvalidOperationException("Upload failed without a result.");
+        int queuedCount = queue.Enqueue(json);
+        UploadHealth queuedHealth = new()
+        {
+            LastUploadAttemptUtc = attemptTime,
+            LastSuccessfulUploadUtc = health.LastSuccessfulUploadUtc,
+            LastUploadStatus = "queued",
+            LastUploadError = failure.Message,
+            QueuedReportCount = queuedCount
+        };
+        queue.SaveHealth(queuedHealth);
+        WriteReportRuntime(reportFullPath, parsedReport, queuedHealth, true);
+
+        Console.WriteLine($"Upload deferred ({failure.StatusCode}): {failure.Message}");
+        Console.WriteLine($"Queued reports: {queuedCount}");
+        Console.WriteLine("The report will be retried automatically on the next scheduled run.");
+        return 0;
+    }
+
+    private static void WriteReportRuntime(string reportPath, ScanResult? report, UploadHealth health, bool attempted)
+    {
+        if (report is null)
+        {
+            return;
+        }
+
+        ScanResult updated = report with
+        {
+            Runtime = report.Runtime with
+            {
+                UploadAttempted = attempted,
+                UploadSucceeded = string.Equals(health.LastUploadStatus, "ok", StringComparison.Ordinal),
+                UploadStatus = health.LastUploadStatus,
+                QueuedReportCount = health.QueuedReportCount,
+                LastSuccessfulUploadUtc = health.LastSuccessfulUploadUtc,
+                LastUploadError = health.LastUploadError
+            }
+        };
+        File.WriteAllText(reportPath, JsonSerializer.Serialize(updated, JsonOptions.Default));
+    }
+
     private static string ReadValue(string[] args, ref int index, string argName)
     {
         if (index + 1 >= args.Length)

src/OCSentinelCli/Configuration/ClientConfiguration.cs

@@ -18,6 +18,8 @@ internal sealed record ClientConfiguration
 
     public int UploadTimeoutSeconds { get; init; } = 30;
 
+    public int UploadQueueMaxReports { get; init; } = 100;
+
     public bool EnableVulnerabilityCorrelation { get; init; } = true;
 
     public string VulnerabilityCsvPath { get; init; } = string.Empty;

src/OCSentinelCli/Models.cs

@@ -136,6 +136,16 @@ internal sealed record ScanRuntimeMetadata
     public DateTimeOffset FinishedAtUtc { get; init; }
 
     public bool UploadAttempted { get; init; }
+
+    public bool UploadSucceeded { get; init; }
+
+    public string UploadStatus { get; init; } = "not-attempted";
+
+    public int QueuedReportCount { get; init; }
+
+    public DateTimeOffset? LastSuccessfulUploadUtc { get; init; }
+
+    public string LastUploadError { get; init; } = string.Empty;
 }
 
 internal sealed record VulnerabilityFinding

src/OCSentinelCli/OCSentinelCli.csproj

@@ -9,10 +9,10 @@
     <RootNamespace>OCSentinelCli</RootNamespace>
     <Product>OfficeCom Sentinel</Product>
     <Company>OfficeCom</Company>
-    <Version>1.3.2</Version>
+    <Version>1.3.3</Version>
-    <AssemblyVersion>1.3.2.0</AssemblyVersion>
+    <AssemblyVersion>1.3.3.0</AssemblyVersion>
-    <FileVersion>1.3.2.0</FileVersion>
+    <FileVersion>1.3.3.0</FileVersion>
-    <InformationalVersion>1.3.2</InformationalVersion>
+    <InformationalVersion>1.3.3</InformationalVersion>
   </PropertyGroup>
 
   <ItemGroup>

src/OCSentinelCli/Transport/N8nUploadClient.cs

@@ -38,6 +38,8 @@ internal sealed class N8nUploadClient
         request.Headers.Add("X-ATN-Payload-SHA256", payloadHash);
         request.Headers.Add("X-ATN-Signature", signature);
 
+        try
+        {
             using HttpResponseMessage response = httpClient.Send(request);
             string responseText = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
 
@@ -50,6 +52,18 @@ internal sealed class N8nUploadClient
                 PayloadSha256 = payloadHash
             };
         }
+        catch (Exception exception) when (exception is HttpRequestException or TaskCanceledException)
+        {
+            return new UploadResult
+            {
+                Success = false,
+                StatusCode = 0,
+                Message = exception.Message,
+                Nonce = nonce,
+                PayloadSha256 = payloadHash
+            };
+        }
+    }
 
     private static string ComputeSha256(string value)
     {

src/OCSentinelCli/Transport/UploadQueue.cs

@@ -0,0 +1,106 @@
+using System.Text.Json;
+using OCSentinelCli.Models;
+
+namespace OCSentinelCli.Transport;
+
+internal sealed record UploadHealth
+{
+    public DateTimeOffset? LastUploadAttemptUtc { get; init; }
+
+    public DateTimeOffset? LastSuccessfulUploadUtc { get; init; }
+
+    public string LastUploadStatus { get; init; } = "not-attempted";
+
+    public string LastUploadError { get; init; } = string.Empty;
+
+    public int QueuedReportCount { get; init; }
+}
+
+internal sealed class UploadQueue
+{
+    private readonly string queueDirectory;
+    private readonly string healthPath;
+    private readonly int maxReports;
+
+    public UploadQueue(int maxReports)
+    {
+        string root = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "OCSentinel");
+        queueDirectory = Path.Combine(root, "upload-queue");
+        healthPath = Path.Combine(root, "state", "upload-health.json");
+        this.maxReports = Math.Clamp(maxReports, 10, 500);
+    }
+
+    public UploadHealth LoadHealth()
+    {
+        try
+        {
+            if (!File.Exists(healthPath))
+            {
+                return new UploadHealth { QueuedReportCount = GetQueueDepth() };
+            }
+
+            UploadHealth? health = JsonSerializer.Deserialize<UploadHealth>(File.ReadAllText(healthPath), JsonOptions.Default);
+            return (health ?? new UploadHealth()) with { QueuedReportCount = GetQueueDepth() };
+        }
+        catch (Exception)
+        {
+            return new UploadHealth { QueuedReportCount = GetQueueDepth() };
+        }
+    }
+
+    public void SaveHealth(UploadHealth health)
+    {
+        Directory.CreateDirectory(Path.GetDirectoryName(healthPath)!);
+        WriteAtomically(healthPath, JsonSerializer.Serialize(health, JsonOptions.Default));
+    }
+
+    public UploadResult? Drain(N8nUploadClient client, string webhookUrl, string machineName, string clientVersion, string secret, int timeoutSeconds)
+    {
+        foreach (string path in GetQueuedPaths())
+        {
+            string payload = File.ReadAllText(path);
+            UploadResult result = client.UploadJson(webhookUrl, machineName, clientVersion, payload, secret, timeoutSeconds);
+            if (!result.Success)
+            {
+                return result;
+            }
+
+            File.Delete(path);
+        }
+
+        return null;
+    }
+
+    public int Enqueue(string payloadJson)
+    {
+        Directory.CreateDirectory(queueDirectory);
+        string fileName = $"{DateTimeOffset.UtcNow:yyyyMMddHHmmssfff}-{Guid.NewGuid():N}.json";
+        WriteAtomically(Path.Combine(queueDirectory, fileName), payloadJson);
+
+        foreach (string stalePath in GetQueuedPaths().Take(Math.Max(0, GetQueueDepth() - maxReports)))
+        {
+            File.Delete(stalePath);
+        }
+
+        return GetQueueDepth();
+    }
+
+    public int GetQueueDepth()
+    {
+        return Directory.Exists(queueDirectory) ? Directory.EnumerateFiles(queueDirectory, "*.json").Count() : 0;
+    }
+
+    private IEnumerable<string> GetQueuedPaths()
+    {
+        return Directory.Exists(queueDirectory)
+            ? Directory.EnumerateFiles(queueDirectory, "*.json").OrderBy(static path => path, StringComparer.Ordinal)
+            : Enumerable.Empty<string>();
+    }
+
+    private static void WriteAtomically(string path, string content)
+    {
+        string temporaryPath = path + ".tmp";
+        File.WriteAllText(temporaryPath, content);
+        File.Move(temporaryPath, path, true);
+    }
+}