502 lines
18 KiB
C#
502 lines
18 KiB
C#
using Serilog;
|
||
using System;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
using RunRedis.Services;
|
||
using RunRedis.Models;
|
||
using System.Collections.Generic;
|
||
using System.Diagnostics;
|
||
using System.Linq;
|
||
using System.IO;
|
||
|
||
namespace RunRedis
|
||
{
|
||
public class EnhancedWorker
|
||
{
|
||
private readonly RedisSetting _settings;
|
||
private readonly RedisProcessManager _processManager;
|
||
private readonly CancellationTokenSource _cancellationTokenSource;
|
||
private bool _isRunning = false;
|
||
private DateTime _lastHealthCheck = DateTime.MinValue;
|
||
|
||
public EnhancedWorker(RedisSetting settings)
|
||
{
|
||
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
|
||
_processManager = new RedisProcessManager(_settings);
|
||
_cancellationTokenSource = new CancellationTokenSource();
|
||
}
|
||
|
||
public async Task ExecuteAsync()
|
||
{
|
||
_isRunning = true;
|
||
Log.Information("🚀 启动简化 Redis 监控服务");
|
||
|
||
try
|
||
{
|
||
// Simplified initial setup
|
||
await EnsureRedisRunningAsync();
|
||
|
||
// Start simplified monitoring loop
|
||
await SimplifiedMonitoringLoopAsync(_cancellationTokenSource.Token);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Fatal("❌ Redis 监控服务发生严重错误: {Error}", ex.Message);
|
||
throw;
|
||
}
|
||
finally
|
||
{
|
||
_isRunning = false;
|
||
Log.Information("⏹️ Redis 监控服务已停止");
|
||
}
|
||
}
|
||
|
||
public void Stop()
|
||
{
|
||
Log.Information("🛑 正在停止 Redis 监控服务");
|
||
_cancellationTokenSource.Cancel();
|
||
}
|
||
|
||
private async Task EnsureRedisRunningAsync()
|
||
{
|
||
Log.Information("🔧 确保 Redis 正在运行");
|
||
|
||
try
|
||
{
|
||
if (!_processManager.IsRedisRunning())
|
||
{
|
||
Log.Information("⚠️ Redis 未运行，正在启动...");
|
||
var startResult = await _processManager.StartRedisAsync();
|
||
if (!startResult)
|
||
{
|
||
Log.Error("❌ 启动 Redis 失败，执行强制重启");
|
||
await ForceKillAndRestartRedisAsync();
|
||
}
|
||
else
|
||
{
|
||
Log.Information("✅ Redis 启动成功");
|
||
}
|
||
}
|
||
else
|
||
{
|
||
Log.Information("✅ Redis 已在运行");
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ 确保 Redis 运行时发生错误: {Error}", ex.Message);
|
||
await ForceKillAndRestartRedisAsync();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 强制终止所有 Redis 进程并快速重启
|
||
/// </summary>
|
||
private async Task ForceKillAndRestartRedisAsync()
|
||
{
|
||
try
|
||
{
|
||
Log.Warning("<22> 执行强制 Redis 进程终止和重启");
|
||
|
||
// 1. 立即强制终止所有 Redis 进程
|
||
await ForceKillAllRedisProcessesAsync();
|
||
|
||
// 2. 短暂等待系统清理
|
||
await Task.Delay(2000);
|
||
|
||
// 3. 快速重启
|
||
Log.Information("🚀 快速重启 Redis");
|
||
var startResult = await _processManager.StartRedisAsync();
|
||
|
||
if (!startResult)
|
||
{
|
||
Log.Error("❌ 快速重启失败，再次尝试");
|
||
await Task.Delay(1000);
|
||
await _processManager.StartRedisAsync();
|
||
}
|
||
else
|
||
{
|
||
Log.Information("✅ Redis 强制重启成功");
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ 强制重启过程中发生错误: {Error}", ex.Message);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 强制终止所有 Redis 相关进程
|
||
/// </summary>
|
||
private async Task ForceKillAllRedisProcessesAsync()
|
||
{
|
||
try
|
||
{
|
||
Log.Warning("<22> 强制终止所有 Redis 进程");
|
||
|
||
// 获取所有 Redis 相关进程
|
||
var redisProcessNames = new[] { "redis-server", "redis-cli", "redis-sentinel" };
|
||
|
||
foreach (var processName in redisProcessNames)
|
||
{
|
||
var processes = Process.GetProcessesByName(processName);
|
||
foreach (var process in processes)
|
||
{
|
||
try
|
||
{
|
||
if (!process.HasExited)
|
||
{
|
||
Log.Information("🔪 强制终止进程: {ProcessName} (PID: {ProcessId})", processName, process.Id);
|
||
process.Kill(entireProcessTree: true); // 终止整个进程树
|
||
|
||
// 等待进程完全终止
|
||
if (!process.WaitForExit(5000))
|
||
{
|
||
Log.Warning("⚠️ 进程 {ProcessId} 未在5秒内终止", process.Id);
|
||
}
|
||
else
|
||
{
|
||
Log.Information("✅ 进程 {ProcessId} 已终止", process.Id);
|
||
}
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Warning("⚠️ 终止进程 {ProcessId} 时发生错误: {Error}", process.Id, ex.Message);
|
||
}
|
||
finally
|
||
{
|
||
process.Dispose();
|
||
}
|
||
}
|
||
}
|
||
|
||
// 额外清理：使用系统命令强制终止
|
||
try
|
||
{
|
||
await ExecuteSystemCommandAsync("taskkill /f /im redis-server.exe");
|
||
await ExecuteSystemCommandAsync("taskkill /f /im redis-cli.exe");
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Debug("系统命令清理失败: {Error}", ex.Message);
|
||
}
|
||
|
||
Log.Information("✅ Redis 进程强制终止完成");
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ 强制终止 Redis 进程时发生错误: {Error}", ex.Message);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 执行系统命令
|
||
/// </summary>
|
||
private async Task ExecuteSystemCommandAsync(string command)
|
||
{
|
||
try
|
||
{
|
||
var processInfo = new ProcessStartInfo("cmd.exe", $"/c {command}")
|
||
{
|
||
CreateNoWindow = true,
|
||
UseShellExecute = false,
|
||
RedirectStandardOutput = true,
|
||
RedirectStandardError = true
|
||
};
|
||
|
||
using var process = Process.Start(processInfo);
|
||
if (process != null)
|
||
{
|
||
await process.WaitForExitAsync();
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Debug("执行系统命令失败: {Command}, 错误: {Error}", command, ex.Message);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 简化的监控循环
|
||
/// </summary>
|
||
private async Task SimplifiedMonitoringLoopAsync(CancellationToken cancellationToken)
|
||
{
|
||
Log.Information("🔄 启动简化监控循环 (检查间隔: {Interval}秒)", _settings.ProcessCheckIntervalSeconds);
|
||
|
||
while (!cancellationToken.IsCancellationRequested)
|
||
{
|
||
try
|
||
{
|
||
// 简单检查：Redis 是否运行
|
||
if (!_processManager.IsRedisRunning())
|
||
{
|
||
Log.Warning("⚠️ Redis 未运行，执行强制重启");
|
||
await ForceKillAndRestartRedisAsync();
|
||
}
|
||
else
|
||
{
|
||
// 简化的健康检查
|
||
await PerformSimplifiedHealthCheckAsync();
|
||
}
|
||
|
||
// 等待下次检查
|
||
await Task.Delay(_settings.ProcessCheckIntervalSeconds * 1000, cancellationToken);
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
Log.Information("🛑 监控循环已取消");
|
||
break;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ 监控循环错误: {Error}", ex.Message);
|
||
// 发生任何错误都执行强制重启
|
||
await ForceKillAndRestartRedisAsync();
|
||
await Task.Delay(3000, cancellationToken);
|
||
}
|
||
}
|
||
|
||
Log.Information("⏹️ 监控循环已停止");
|
||
}
|
||
|
||
/// <summary>
|
||
/// 简化的健康检查 - 检查连接、数据读写功能和性能
|
||
/// </summary>
|
||
private async Task PerformSimplifiedHealthCheckAsync()
|
||
{
|
||
try
|
||
{
|
||
var timeSinceLastCheck = DateTime.Now - _lastHealthCheck;
|
||
if (timeSinceLastCheck.TotalSeconds < _settings.HealthCheckIntervalSeconds)
|
||
{
|
||
return; // 还没到检查时间
|
||
}
|
||
|
||
_lastHealthCheck = DateTime.Now;
|
||
|
||
// 执行增强的健康检查，包含超时机制
|
||
var healthCheckPassed = await PerformEnhancedHealthCheckWithTimeoutAsync();
|
||
if (!healthCheckPassed)
|
||
{
|
||
Log.Warning("⚠️ Redis 健康检查失败，执行强制重启");
|
||
await ForceKillAndRestartRedisAsync();
|
||
return;
|
||
}
|
||
|
||
Log.Information("✅ Redis 健康检查通过");
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ 健康检查失败: {Error}", ex.Message);
|
||
// 健康检查失败就强制重启
|
||
await ForceKillAndRestartRedisAsync();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 执行增强的健康检查，包含连接测试、数据读写测试和超时机制
|
||
/// </summary>
|
||
private async Task<bool> PerformEnhancedHealthCheckWithTimeoutAsync()
|
||
{
|
||
const int timeoutSeconds = 5;
|
||
var testKey = $"health_check_{DateTime.Now:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
|
||
var testValue = $"test_value_{DateTime.Now.Ticks}";
|
||
|
||
try
|
||
{
|
||
Log.Debug("🔍 开始增强健康检查 - 测试键: {TestKey}", testKey);
|
||
|
||
// 使用CancellationToken实现5秒超时
|
||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds));
|
||
|
||
try
|
||
{
|
||
// 1. 基本连接测试
|
||
var pingTask = TestRedisConnectionAsync();
|
||
var pingResult = await ExecuteWithTimeoutAsync(pingTask, cts.Token);
|
||
|
||
if (!pingResult)
|
||
{
|
||
Log.Warning("⚠️ Redis PING 测试失败");
|
||
return false;
|
||
}
|
||
Log.Debug("✅ Redis PING 测试通过");
|
||
|
||
// 2. 数据写入测试
|
||
var writeTask = TestRedisWriteAsync(testKey, testValue);
|
||
var writeResult = await ExecuteWithTimeoutAsync(writeTask, cts.Token);
|
||
|
||
if (!writeResult)
|
||
{
|
||
Log.Warning("⚠️ Redis 数据写入测试失败");
|
||
return false;
|
||
}
|
||
Log.Debug("✅ Redis 数据写入测试通过");
|
||
|
||
// 3. 数据读取测试
|
||
var readTask = TestRedisReadAsync(testKey, testValue);
|
||
var readResult = await ExecuteWithTimeoutAsync(readTask, cts.Token);
|
||
|
||
if (!readResult)
|
||
{
|
||
Log.Warning("⚠️ Redis 数据读取测试失败");
|
||
return false;
|
||
}
|
||
Log.Debug("✅ Redis 数据读取测试通过");
|
||
|
||
return true;
|
||
}
|
||
catch (OperationCanceledException) when (cts.Token.IsCancellationRequested)
|
||
{
|
||
Log.Warning("⚠️ Redis 健康检查超时 ({TimeoutSeconds}秒)，操作响应过慢", timeoutSeconds);
|
||
return false;
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ 增强健康检查过程中发生错误: {Error}", ex.Message);
|
||
return false;
|
||
}
|
||
finally
|
||
{
|
||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||
await CleanupTestDataAsync(testKey, cts.Token); // 需修改方法支持CancellationToken
|
||
// 清理测试数据
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 执行带超时的任务 (.NET Core 3.1 兼容版本)
|
||
/// </summary>
|
||
private async Task<T> ExecuteWithTimeoutAsync<T>(Task<T> task, CancellationToken cancellationToken)
|
||
{
|
||
var tcs = new TaskCompletionSource<T>();
|
||
|
||
// 注册取消回调
|
||
using var registration = cancellationToken.Register(() =>
|
||
tcs.TrySetCanceled(cancellationToken));
|
||
|
||
// 等待原始任务完成
|
||
_ = task.ContinueWith(t =>
|
||
{
|
||
if (t.IsFaulted)
|
||
tcs.TrySetException(t.Exception.InnerException ?? t.Exception);
|
||
else if (t.IsCanceled)
|
||
tcs.TrySetCanceled();
|
||
else
|
||
tcs.TrySetResult(t.Result);
|
||
}, TaskContinuationOptions.ExecuteSynchronously);
|
||
|
||
return await tcs.Task;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 测试 Redis 数据写入功能
|
||
/// </summary>
|
||
private async Task<bool> TestRedisWriteAsync(string key, string value)
|
||
{
|
||
try
|
||
{
|
||
var command = $"SET {key} \"{value}\"";
|
||
var result = await _processManager.ExecuteRedisCliCommandAsync(command);
|
||
|
||
var success = !string.IsNullOrEmpty(result) && result.Contains("OK");
|
||
if (success)
|
||
{
|
||
Log.Debug("✅ Redis 写入操作成功: {Key} = {Value}", key, value);
|
||
}
|
||
else
|
||
{
|
||
Log.Warning("⚠️ Redis 写入操作失败: {Result}", result);
|
||
}
|
||
|
||
return success;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ Redis 写入测试异常: {Error}", ex.Message);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 测试 Redis 数据读取功能
|
||
/// </summary>
|
||
private async Task<bool> TestRedisReadAsync(string key, string expectedValue)
|
||
{
|
||
try
|
||
{
|
||
var command = $"GET {key}";
|
||
|
||
var result = await _processManager.ExecuteRedisCliCommandAsync(command);
|
||
|
||
var success = !string.IsNullOrEmpty(result) && result.Trim().Trim('"') == expectedValue;
|
||
if (success)
|
||
{
|
||
Log.Debug("✅ Redis 读取操作成功: {Key} = {Value}", key, result);
|
||
}
|
||
else
|
||
{
|
||
Log.Warning("⚠️ Redis 读取操作失败: 期望值 '{ExpectedValue}', 实际值 '{ActualValue}'", expectedValue, result);
|
||
}
|
||
|
||
return success;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Error("❌ Redis 读取测试异常: {Error}", ex.Message);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 清理测试数据
|
||
/// </summary>
|
||
private async Task CleanupTestDataAsync(string testKey, CancellationToken token)
|
||
{
|
||
try
|
||
{
|
||
if (!string.IsNullOrEmpty(testKey))
|
||
{
|
||
var command = $"DEL {testKey}";
|
||
var result = await _processManager.ExecuteRedisCliCommandAsync(command, token);
|
||
Log.Debug("🧹 清理测试数据: {Key}, 结果: {Result}", testKey, result);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Debug("⚠️ 清理测试数据时发生错误: {Error}", ex.Message);
|
||
// 清理失败不影响主流程，只记录日志
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 测试 Redis 连接
|
||
/// </summary>
|
||
private async Task<bool> TestRedisConnectionAsync()
|
||
{
|
||
try
|
||
{
|
||
// 使用 redis-cli 执行简单的 PING 命令
|
||
var result = await _processManager.ExecuteRedisCliCommandAsync("PING");
|
||
return !string.IsNullOrEmpty(result) && result.Contains("PONG");
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log.Debug("Redis 连接测试失败: {Error}", ex.Message);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
public bool IsRunning => _isRunning;
|
||
|
||
public void Dispose()
|
||
{
|
||
_cancellationTokenSource?.Cancel();
|
||
_processManager?.Dispose();
|
||
_cancellationTokenSource?.Dispose();
|
||
}
|
||
}
|
||
}
|