SOMS/src/YunDa.Quick/RunRedis/Services/RedisProcessManager.cs

760 lines
26 KiB
C#
Raw Normal View History

2025-06-17 15:20:33 +08:00
using Serilog;
using StackExchange.Redis;
2025-06-17 15:20:33 +08:00
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
2025-06-17 15:20:33 +08:00
using System.Net.Sockets;
using System.Text;
using System.Threading;
2025-06-17 15:20:33 +08:00
using System.Threading.Tasks;
namespace RunRedis.Services
{
public class RedisProcessManager
{
private readonly RedisSetting _settings;
private Process _redisProcess;
private int _restartAttempts = 0;
public RedisProcessManager(RedisSetting settings)
{
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
}
public bool IsRedisRunning()
{
try
{
// First check if process is running
if (!IsRedisProcessRunning())
return false;
// Then check if Redis is responding on the configured port
using (var tcpClient = new TcpClient())
{
var connectTask = tcpClient.ConnectAsync(_settings.Host, int.Parse(_settings.Port));
var timeoutTask = Task.Delay(_settings.ConnectionTimeoutSeconds * 1000);
var completedTask = Task.WaitAny(connectTask, timeoutTask);
if (completedTask == 0 && tcpClient.Connected)
{
return true;
}
}
return false;
}
catch (Exception ex)
{
Log.Debug("Error checking Redis status: {Error}", ex.Message);
return false;
}
}
2025-06-17 15:20:33 +08:00
public bool IsRedisProcessRunning()
{
try
{
var processes = Process.GetProcessesByName("redis-server");
return processes.Length > 0;
}
catch
{
return false;
}
}
public async Task<bool> StartRedisAsync()
{
try
{
if (IsRedisRunning())
{
Log.Information("Redis is already running");
return true;
}
// Prepare for startup
await PrepareForStartupAsync();
// Determine Redis executable path
string redisExecutable = GetRedisExecutablePath();
if (!File.Exists(redisExecutable))
{
Log.Error("Redis executable not found at: {Path}", redisExecutable);
return false;
}
// Create process start info
var startInfo = new ProcessStartInfo
{
FileName = redisExecutable,
WorkingDirectory = Path.GetDirectoryName(redisExecutable),
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true
};
// Add configuration file if exists
string configPath = GetRedisConfigPath();
if (File.Exists(configPath))
{
startInfo.Arguments = $"\"{configPath}\"";
Log.Information("Starting Redis with config file: {ConfigPath}", configPath);
}
else
{
// Use command line arguments
startInfo.Arguments = BuildRedisArguments();
Log.Information("Starting Redis with command line arguments");
}
_redisProcess = Process.Start(startInfo);
if (_redisProcess != null)
{
Log.Information("Redis process started with PID: {ProcessId}", _redisProcess.Id);
// Wait a moment for the process to initialize
await Task.Delay(5000);
// Verify the process is still running and Redis is responding
if (!_redisProcess.HasExited && IsRedisRunning())
{
_restartAttempts = 0; // Reset restart attempts on successful start
Log.Information("Redis started successfully");
return true;
}
else
{
Log.Error("Redis process exited immediately after start or is not responding");
return false;
}
}
else
{
Log.Error("Failed to start Redis process");
return false;
}
}
catch (Exception ex)
{
Log.Error("Error starting Redis: {Error}", ex.Message);
return false;
}
}
public async Task<bool> StopRedisAsync(bool graceful = true)
{
try
{
if (!IsRedisProcessRunning())
{
Log.Information("Redis is not running");
return true;
}
if (graceful)
{
// Try graceful shutdown using redis-cli
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await ExecuteRedisCliCommandAsync("SHUTDOWN SAVE", cts.Token);
2025-06-17 15:20:33 +08:00
await Task.Delay(5000); // Wait for graceful shutdown
}
// If still running, force kill
var processes = Process.GetProcessesByName("redis-server");
foreach (var process in processes)
{
try
{
if (!process.HasExited)
{
process.Kill();
await process.WaitForExitAsync();
Log.Information("Redis process {ProcessId} terminated", process.Id);
}
}
finally
{
process.Dispose();
}
}
return true;
}
catch (Exception ex)
{
Log.Error("Error stopping Redis: {Error}", ex.Message);
return false;
}
}
public async Task<bool> RestartRedisAsync()
{
try
2025-06-17 15:20:33 +08:00
{
// 2. 优雅关闭Redis
Log.Information("🛑 Gracefully stopping Redis...");
await StopRedisAsync(graceful: true);
// 3. 清理可能的锁文件和临时文件
await CleanupStaleFilesAsync();
// 4. 验证数据文件完整性
await ValidateDataFilesAsync();
// 5. 等待系统稳定
Log.Information("⏳ Waiting {Delay} seconds before restart...", _settings.RestartDelaySeconds);
await Task.Delay(_settings.RestartDelaySeconds * 1000);
// 6. 启动Redis
Log.Information("🚀 Starting Redis...");
var startResult = await StartRedisAsync();
if (startResult)
{
// 7. 验证重启后的状态
await Task.Delay(3000); // 等待Redis完全启动
var isHealthy = await VerifyRedisHealthAfterRestartAsync();
2025-06-17 15:20:33 +08:00
if (isHealthy)
{
Log.Information("✅ Redis restart successful and healthy");
_restartAttempts = 0; // 重置重启计数
return true;
}
else
{
Log.Warning("⚠️ Redis started but health check failed");
return false;
}
}
else
{
Log.Error("❌ Failed to start Redis after restart attempt");
return false;
}
}
catch (Exception ex)
{
Log.Error("❌ Error during Redis restart: {Error}", ex.Message);
return false;
}
2025-06-17 15:20:33 +08:00
}
private async Task PrepareForStartupAsync()
{
try
{
// Ensure data directory exists
if (!string.IsNullOrEmpty(_settings.DataDirectory))
{
Directory.CreateDirectory(_settings.DataDirectory);
}
// Ensure log directory exists
if (!string.IsNullOrEmpty(_settings.LogDirectory))
{
Directory.CreateDirectory(_settings.LogDirectory);
}
// Clean up any stale lock files or temporary files
await CleanupStaleFilesAsync();
}
catch (Exception ex)
{
Log.Warning("Error during startup preparation: {Error}", ex.Message);
}
}
private Task CleanupStaleFilesAsync()
{
try
{
// Remove Redis dump.rdb lock files if they exist
var lockFiles = new[] { "dump.rdb.lock", "redis.pid" };
foreach (var lockFile in lockFiles)
{
var lockPath = Path.Combine(_settings.DataDirectory, lockFile);
if (File.Exists(lockPath))
{
File.Delete(lockPath);
Log.Information("Removed stale lock file: {LockFile}", lockPath);
}
}
}
catch (Exception ex)
{
Log.Warning("Error cleaning up stale files: {Error}", ex.Message);
}
return Task.CompletedTask;
}
private string GetRedisExecutablePath()
{
// Try configured path first
if (!string.IsNullOrEmpty(_settings.RedisExecutablePath) && File.Exists(_settings.RedisExecutablePath))
{
return _settings.RedisExecutablePath;
}
// Try current directory
var currentDirPath = Path.Combine(Directory.GetCurrentDirectory(), "redis-server.exe");
if (File.Exists(currentDirPath))
{
return currentDirPath;
}
// Try PATH environment variable
return "redis-server.exe"; // Let the system find it
}
private string GetRedisConfigPath()
{
// Try configured path first
if (!string.IsNullOrEmpty(_settings.RedisConfigPath) && File.Exists(_settings.RedisConfigPath))
{
return _settings.RedisConfigPath;
}
// Try current directory
var currentDirPath = Path.Combine(Directory.GetCurrentDirectory(), "redis.conf");
if (File.Exists(currentDirPath))
{
return currentDirPath;
}
return null; // No config file found
}
private string BuildRedisArguments()
{
var args = new System.Collections.Generic.List<string>();
// Basic configuration
args.Add($"--port {_settings.Port}");
args.Add($"--bind {_settings.Host}");
if (!string.IsNullOrEmpty(_settings.Auth))
{
args.Add($"--requirepass {_settings.Auth}");
}
if (!string.IsNullOrEmpty(_settings.DataDirectory))
{
args.Add($"--dir \"{_settings.DataDirectory}\"");
}
// Memory settings
if (_settings.MaxMemoryUsageMB > 0)
{
args.Add($"--maxmemory {_settings.MaxMemoryUsageMB}mb");
args.Add($"--maxmemory-policy {_settings.MaxMemoryPolicy}");
}
// Persistence settings
if (_settings.EnablePersistence)
{
args.Add($"--save {_settings.SaveConfiguration}");
}
else
{
args.Add("--save \"\"");
}
// Database settings
args.Add($"--databases {_settings.DatabaseCount}");
args.Add($"--timeout {_settings.ClientTimeoutSeconds}");
// Slow log settings
if (_settings.EnableSlowLog)
{
args.Add($"--slowlog-max-len {_settings.SlowLogMaxLength}");
args.Add($"--slowlog-log-slower-than {_settings.SlowLogSlowerThanMicroseconds}");
}
return string.Join(" ", args);
}
public async Task<string> ExecuteRedisCliCommandAsync(string command, CancellationToken token =default)
2025-06-17 15:20:33 +08:00
{
Process process = null;
2025-06-17 15:20:33 +08:00
try
{
if (token == default)
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
token = cts.Token;
}
2025-06-17 15:20:33 +08:00
string redisCliPath = GetRedisCliPath();
2025-06-17 15:20:33 +08:00
var processStartInfo = new ProcessStartInfo
{
FileName = redisCliPath,
Arguments = BuildRedisCliArguments(command),
2025-06-17 15:20:33 +08:00
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true
};
process = Process.Start(processStartInfo);
var outputBuilder = new StringBuilder();
var errorBuilder = new StringBuilder();
// 异步读取输出和错误
var outputTask = ReadStreamAsync(process.StandardOutput, outputBuilder);
var errorTask = ReadStreamAsync(process.StandardError, errorBuilder);
// 创建超时监控任务
var processExitTask = process.WaitForExitAsync();
var timeoutTask = Task.Delay(Timeout.Infinite, token);
2025-06-17 15:20:33 +08:00
// 等待任意任务完成
var completedTask = await Task.WhenAny(processExitTask, timeoutTask);
2025-06-17 15:20:33 +08:00
if (completedTask == timeoutTask)
2025-06-17 15:20:33 +08:00
{
// 超时处理
try
2025-06-17 15:20:33 +08:00
{
if (!process.HasExited)
{
process.Kill();
Log.Warning("Redis CLI command timed out and was killed: {Command}", command);
}
2025-06-17 15:20:33 +08:00
}
catch (InvalidOperationException) { /* 进程已退出 */ }
token.ThrowIfCancellationRequested();
}
// 确保获取退出代码
await processExitTask;
await Task.WhenAll(outputTask, errorTask);
var output = outputBuilder.ToString().Trim();
var error = errorBuilder.ToString().Trim();
if (process.ExitCode == 0)
{
Log.Debug("Redis CLI command executed successfully: {Command}", command);
return output;
2025-06-17 15:20:33 +08:00
}
Log.Warning("Redis CLI command failed (Exit {ExitCode}): {Command}, Error: {Error}",
process.ExitCode, command, error);
return null;
}
catch (OperationCanceledException)
{
Log.Warning("Redis CLI command canceled: {Command}", command);
throw; // 重新抛出让调用方处理
2025-06-17 15:20:33 +08:00
}
catch (Exception ex)
{
Log.Error(ex, "Error executing Redis CLI command: {Command}", command);
2025-06-17 15:20:33 +08:00
return null;
}
finally
{
process?.Dispose();
}
}
// 辅助方法构建Redis CLI参数
private string BuildRedisCliArguments(string command)
{
var args = new StringBuilder($"-h {_settings.Host} -p {_settings.Port}");
if (!string.IsNullOrEmpty(_settings.Auth))
{
args.Append($" -a {_settings.Auth}");
}
args.Append($" {command}");
return args.ToString();
}
// 辅助方法:异步读取流
private async Task ReadStreamAsync(StreamReader reader, StringBuilder builder)
{
char[] buffer = new char[4096];
int bytesRead;
while ((bytesRead = await reader.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
builder.Append(buffer, 0, bytesRead);
}
2025-06-17 15:20:33 +08:00
}
2025-06-17 15:20:33 +08:00
private string GetRedisCliPath()
{
// Try configured path first
if (!string.IsNullOrEmpty(_settings.RedisCliPath) && File.Exists(_settings.RedisCliPath))
{
return _settings.RedisCliPath;
}
// Try current directory
var currentDirPath = Path.Combine(Directory.GetCurrentDirectory(), "redis-cli.exe");
if (File.Exists(currentDirPath))
{
return currentDirPath;
}
// Try PATH environment variable
return "redis-cli.exe"; // Let the system find it
}
private async Task CreateEmergencyBackupAsync()
{
try
{
if (!_settings.EnableAutoBackup)
return;
Log.Information("💾 Creating emergency backup before restart...");
// 确保备份目录存在
if (!Directory.Exists(_settings.BackupDirectory))
{
Directory.CreateDirectory(_settings.BackupDirectory);
}
// 尝试触发BGSAVE如果Redis还在运行
if (IsRedisRunning())
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var saveResult = await ExecuteRedisCliCommandAsync("BGSAVE", cts.Token);
if (!string.IsNullOrEmpty(saveResult) && !saveResult.Contains("ERR"))
{
Log.Information("✅ Background save triggered successfully");
await Task.Delay(2000); // 等待保存完成
}
}
// 复制现有的数据文件
var dataFiles = new[] { "dump.rdb", "appendonly.aof" };
var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
foreach (var dataFile in dataFiles)
{
var sourcePath = Path.Combine(_settings.DataDirectory, dataFile);
if (File.Exists(sourcePath))
{
var backupPath = Path.Combine(_settings.BackupDirectory, $"emergency_{dataFile}_{timestamp}");
File.Copy(sourcePath, backupPath, true);
Log.Information("📁 Backed up {DataFile} to {BackupPath}", dataFile, backupPath);
}
}
}
catch (Exception ex)
{
Log.Warning("⚠️ Failed to create emergency backup: {Error}", ex.Message);
}
}
private async Task ValidateDataFilesAsync()
{
try
{
Log.Information("🔍 Validating Redis data files...");
var dumpFile = Path.Combine(_settings.DataDirectory, "dump.rdb");
var aofFile = Path.Combine(_settings.DataDirectory, "appendonly.aof");
// 检查RDB文件
if (File.Exists(dumpFile))
{
var fileInfo = new FileInfo(dumpFile);
if (fileInfo.Length == 0)
{
Log.Warning("⚠️ RDB file is empty, may be corrupted");
}
else
{
Log.Information("✅ RDB file exists and has data ({Size} bytes)", fileInfo.Length);
}
}
// 检查AOF文件
if (File.Exists(aofFile))
{
var fileInfo = new FileInfo(aofFile);
Log.Information("✅ AOF file exists ({Size} bytes)", fileInfo.Length);
}
// 检查并修复可能的权限问题
await EnsureDataDirectoryPermissionsAsync();
}
catch (Exception ex)
{
Log.Warning("⚠️ Error validating data files: {Error}", ex.Message);
}
}
private Task EnsureDataDirectoryPermissionsAsync()
{
try
{
if (!Directory.Exists(_settings.DataDirectory))
{
Directory.CreateDirectory(_settings.DataDirectory);
Log.Information("📁 Created data directory: {DataDirectory}", _settings.DataDirectory);
}
// 确保目录可写
var testFile = Path.Combine(_settings.DataDirectory, "test_write.tmp");
File.WriteAllText(testFile, "test");
File.Delete(testFile);
Log.Debug("✅ Data directory permissions verified");
}
catch (Exception ex)
{
Log.Error("❌ Data directory permission issue: {Error}", ex.Message);
}
return Task.CompletedTask;
}
private async Task<bool> VerifyRedisHealthAfterRestartAsync()
{
try
{
Log.Information("🏥 Verifying Redis health after restart...");
// 等待Redis完全启动
for (int i = 0; i < 10; i++)
{
if (IsRedisRunning())
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
// 测试基本连接
var pingResult = await ExecuteRedisCliCommandAsync("PING", cts.Token);
if (!string.IsNullOrEmpty(pingResult) && pingResult.Contains("PONG"))
{
Log.Information("✅ Redis is responding to PING");
// 测试基本操作
var setResult = await ExecuteRedisCliCommandAsync("SET health_check_key test_value");
var getResult = await ExecuteRedisCliCommandAsync("GET health_check_key");
var delResult = await ExecuteRedisCliCommandAsync("DEL health_check_key");
if (!string.IsNullOrEmpty(getResult) && getResult.Contains("test_value"))
{
Log.Information("✅ Redis basic operations working correctly");
return true;
}
}
}
await Task.Delay(1000);
}
Log.Warning("⚠️ Redis health verification failed after restart");
return false;
}
catch (Exception ex)
{
Log.Error("❌ Error verifying Redis health: {Error}", ex.Message);
return false;
}
}
public async Task<bool> RecoverFromBackupAsync(string backupPath = null)
{
try
{
Log.Information("🔄 Starting Redis data recovery...");
// 停止Redis
await StopRedisAsync(graceful: false);
// 如果没有指定备份路径,寻找最新的备份
if (string.IsNullOrEmpty(backupPath))
{
backupPath = FindLatestBackup();
}
if (string.IsNullOrEmpty(backupPath) || !File.Exists(backupPath))
{
Log.Error("❌ No valid backup found for recovery");
return false;
}
Log.Information("📁 Recovering from backup: {BackupPath}", backupPath);
// 备份当前损坏的文件
var corruptedBackupDir = Path.Combine(_settings.BackupDirectory, $"corrupted_{DateTime.Now:yyyyMMdd_HHmmss}");
Directory.CreateDirectory(corruptedBackupDir);
var currentDumpFile = Path.Combine(_settings.DataDirectory, "dump.rdb");
if (File.Exists(currentDumpFile))
{
File.Move(currentDumpFile, Path.Combine(corruptedBackupDir, "dump.rdb"));
}
// 恢复备份文件
File.Copy(backupPath, currentDumpFile, true);
Log.Information("✅ Backup file restored");
// 重启Redis
var startResult = await StartRedisAsync();
if (startResult)
{
Log.Information("✅ Redis recovery completed successfully");
return true;
}
else
{
Log.Error("❌ Failed to start Redis after recovery");
return false;
}
}
catch (Exception ex)
{
Log.Error("❌ Error during Redis recovery: {Error}", ex.Message);
return false;
}
}
private string FindLatestBackup()
{
try
{
if (!Directory.Exists(_settings.BackupDirectory))
return null;
var backupFiles = Directory.GetFiles(_settings.BackupDirectory, "*.rdb")
.OrderByDescending(f => new FileInfo(f).CreationTime)
.ToArray();
return backupFiles.FirstOrDefault();
}
catch
{
return null;
}
}
2025-06-17 15:20:33 +08:00
public void Dispose()
{
_redisProcess?.Dispose();
}
}
}