SOMS/src/YunDa.Quick/RunRedis/Services/RedisProcessManager.cs
2025-07-04 14:10:12 +08:00

760 lines
26 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

using Serilog;
using StackExchange.Redis;
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace RunRedis.Services
{
public class RedisProcessManager
{
private readonly RedisSetting _settings;
private Process _redisProcess;
private int _restartAttempts = 0;
public RedisProcessManager(RedisSetting settings)
{
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
}
public bool IsRedisRunning()
{
try
{
// First check if process is running
if (!IsRedisProcessRunning())
return false;
// Then check if Redis is responding on the configured port
using (var tcpClient = new TcpClient())
{
var connectTask = tcpClient.ConnectAsync(_settings.Host, int.Parse(_settings.Port));
var timeoutTask = Task.Delay(_settings.ConnectionTimeoutSeconds * 1000);
var completedTask = Task.WaitAny(connectTask, timeoutTask);
if (completedTask == 0 && tcpClient.Connected)
{
return true;
}
}
return false;
}
catch (Exception ex)
{
Log.Debug("Error checking Redis status: {Error}", ex.Message);
return false;
}
}
public bool IsRedisProcessRunning()
{
try
{
var processes = Process.GetProcessesByName("redis-server");
return processes.Length > 0;
}
catch
{
return false;
}
}
public async Task<bool> StartRedisAsync()
{
try
{
if (IsRedisRunning())
{
Log.Information("Redis is already running");
return true;
}
// Prepare for startup
await PrepareForStartupAsync();
// Determine Redis executable path
string redisExecutable = GetRedisExecutablePath();
if (!File.Exists(redisExecutable))
{
Log.Error("Redis executable not found at: {Path}", redisExecutable);
return false;
}
// Create process start info
var startInfo = new ProcessStartInfo
{
FileName = redisExecutable,
WorkingDirectory = Path.GetDirectoryName(redisExecutable),
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true
};
// Add configuration file if exists
string configPath = GetRedisConfigPath();
if (File.Exists(configPath))
{
startInfo.Arguments = $"\"{configPath}\"";
Log.Information("Starting Redis with config file: {ConfigPath}", configPath);
}
else
{
// Use command line arguments
startInfo.Arguments = BuildRedisArguments();
Log.Information("Starting Redis with command line arguments");
}
_redisProcess = Process.Start(startInfo);
if (_redisProcess != null)
{
Log.Information("Redis process started with PID: {ProcessId}", _redisProcess.Id);
// Wait a moment for the process to initialize
await Task.Delay(5000);
// Verify the process is still running and Redis is responding
if (!_redisProcess.HasExited && IsRedisRunning())
{
_restartAttempts = 0; // Reset restart attempts on successful start
Log.Information("Redis started successfully");
return true;
}
else
{
Log.Error("Redis process exited immediately after start or is not responding");
return false;
}
}
else
{
Log.Error("Failed to start Redis process");
return false;
}
}
catch (Exception ex)
{
Log.Error("Error starting Redis: {Error}", ex.Message);
return false;
}
}
public async Task<bool> StopRedisAsync(bool graceful = true)
{
try
{
if (!IsRedisProcessRunning())
{
Log.Information("Redis is not running");
return true;
}
if (graceful)
{
// Try graceful shutdown using redis-cli
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await ExecuteRedisCliCommandAsync("SHUTDOWN SAVE", cts.Token);
await Task.Delay(5000); // Wait for graceful shutdown
}
// If still running, force kill
var processes = Process.GetProcessesByName("redis-server");
foreach (var process in processes)
{
try
{
if (!process.HasExited)
{
process.Kill();
await process.WaitForExitAsync();
Log.Information("Redis process {ProcessId} terminated", process.Id);
}
}
finally
{
process.Dispose();
}
}
return true;
}
catch (Exception ex)
{
Log.Error("Error stopping Redis: {Error}", ex.Message);
return false;
}
}
public async Task<bool> RestartRedisAsync()
{
try
{
// 2. 优雅关闭Redis
Log.Information("🛑 Gracefully stopping Redis...");
await StopRedisAsync(graceful: true);
// 3. 清理可能的锁文件和临时文件
await CleanupStaleFilesAsync();
// 4. 验证数据文件完整性
await ValidateDataFilesAsync();
// 5. 等待系统稳定
Log.Information("⏳ Waiting {Delay} seconds before restart...", _settings.RestartDelaySeconds);
await Task.Delay(_settings.RestartDelaySeconds * 1000);
// 6. 启动Redis
Log.Information("🚀 Starting Redis...");
var startResult = await StartRedisAsync();
if (startResult)
{
// 7. 验证重启后的状态
await Task.Delay(3000); // 等待Redis完全启动
var isHealthy = await VerifyRedisHealthAfterRestartAsync();
if (isHealthy)
{
Log.Information("✅ Redis restart successful and healthy");
_restartAttempts = 0; // 重置重启计数
return true;
}
else
{
Log.Warning("⚠️ Redis started but health check failed");
return false;
}
}
else
{
Log.Error("❌ Failed to start Redis after restart attempt");
return false;
}
}
catch (Exception ex)
{
Log.Error("❌ Error during Redis restart: {Error}", ex.Message);
return false;
}
}
private async Task PrepareForStartupAsync()
{
try
{
// Ensure data directory exists
if (!string.IsNullOrEmpty(_settings.DataDirectory))
{
Directory.CreateDirectory(_settings.DataDirectory);
}
// Ensure log directory exists
if (!string.IsNullOrEmpty(_settings.LogDirectory))
{
Directory.CreateDirectory(_settings.LogDirectory);
}
// Clean up any stale lock files or temporary files
await CleanupStaleFilesAsync();
}
catch (Exception ex)
{
Log.Warning("Error during startup preparation: {Error}", ex.Message);
}
}
private Task CleanupStaleFilesAsync()
{
try
{
// Remove Redis dump.rdb lock files if they exist
var lockFiles = new[] { "dump.rdb.lock", "redis.pid" };
foreach (var lockFile in lockFiles)
{
var lockPath = Path.Combine(_settings.DataDirectory, lockFile);
if (File.Exists(lockPath))
{
File.Delete(lockPath);
Log.Information("Removed stale lock file: {LockFile}", lockPath);
}
}
}
catch (Exception ex)
{
Log.Warning("Error cleaning up stale files: {Error}", ex.Message);
}
return Task.CompletedTask;
}
private string GetRedisExecutablePath()
{
// Try configured path first
if (!string.IsNullOrEmpty(_settings.RedisExecutablePath) && File.Exists(_settings.RedisExecutablePath))
{
return _settings.RedisExecutablePath;
}
// Try current directory
var currentDirPath = Path.Combine(Directory.GetCurrentDirectory(), "redis-server.exe");
if (File.Exists(currentDirPath))
{
return currentDirPath;
}
// Try PATH environment variable
return "redis-server.exe"; // Let the system find it
}
private string GetRedisConfigPath()
{
// Try configured path first
if (!string.IsNullOrEmpty(_settings.RedisConfigPath) && File.Exists(_settings.RedisConfigPath))
{
return _settings.RedisConfigPath;
}
// Try current directory
var currentDirPath = Path.Combine(Directory.GetCurrentDirectory(), "redis.conf");
if (File.Exists(currentDirPath))
{
return currentDirPath;
}
return null; // No config file found
}
private string BuildRedisArguments()
{
var args = new System.Collections.Generic.List<string>();
// Basic configuration
args.Add($"--port {_settings.Port}");
args.Add($"--bind {_settings.Host}");
if (!string.IsNullOrEmpty(_settings.Auth))
{
args.Add($"--requirepass {_settings.Auth}");
}
if (!string.IsNullOrEmpty(_settings.DataDirectory))
{
args.Add($"--dir \"{_settings.DataDirectory}\"");
}
// Memory settings
if (_settings.MaxMemoryUsageMB > 0)
{
args.Add($"--maxmemory {_settings.MaxMemoryUsageMB}mb");
args.Add($"--maxmemory-policy {_settings.MaxMemoryPolicy}");
}
// Persistence settings
if (_settings.EnablePersistence)
{
args.Add($"--save {_settings.SaveConfiguration}");
}
else
{
args.Add("--save \"\"");
}
// Database settings
args.Add($"--databases {_settings.DatabaseCount}");
args.Add($"--timeout {_settings.ClientTimeoutSeconds}");
// Slow log settings
if (_settings.EnableSlowLog)
{
args.Add($"--slowlog-max-len {_settings.SlowLogMaxLength}");
args.Add($"--slowlog-log-slower-than {_settings.SlowLogSlowerThanMicroseconds}");
}
return string.Join(" ", args);
}
public async Task<string> ExecuteRedisCliCommandAsync(string command, CancellationToken token =default)
{
Process process = null;
try
{
if (token == default)
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
token = cts.Token;
}
string redisCliPath = GetRedisCliPath();
var processStartInfo = new ProcessStartInfo
{
FileName = redisCliPath,
Arguments = BuildRedisCliArguments(command),
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true
};
process = Process.Start(processStartInfo);
var outputBuilder = new StringBuilder();
var errorBuilder = new StringBuilder();
// 异步读取输出和错误
var outputTask = ReadStreamAsync(process.StandardOutput, outputBuilder);
var errorTask = ReadStreamAsync(process.StandardError, errorBuilder);
// 创建超时监控任务
var processExitTask = process.WaitForExitAsync();
var timeoutTask = Task.Delay(Timeout.Infinite, token);
// 等待任意任务完成
var completedTask = await Task.WhenAny(processExitTask, timeoutTask);
if (completedTask == timeoutTask)
{
// 超时处理
try
{
if (!process.HasExited)
{
process.Kill();
Log.Warning("Redis CLI command timed out and was killed: {Command}", command);
}
}
catch (InvalidOperationException) { /* 进程已退出 */ }
token.ThrowIfCancellationRequested();
}
// 确保获取退出代码
await processExitTask;
await Task.WhenAll(outputTask, errorTask);
var output = outputBuilder.ToString().Trim();
var error = errorBuilder.ToString().Trim();
if (process.ExitCode == 0)
{
Log.Debug("Redis CLI command executed successfully: {Command}", command);
return output;
}
Log.Warning("Redis CLI command failed (Exit {ExitCode}): {Command}, Error: {Error}",
process.ExitCode, command, error);
return null;
}
catch (OperationCanceledException)
{
Log.Warning("Redis CLI command canceled: {Command}", command);
throw; // 重新抛出让调用方处理
}
catch (Exception ex)
{
Log.Error(ex, "Error executing Redis CLI command: {Command}", command);
return null;
}
finally
{
process?.Dispose();
}
}
// 辅助方法构建Redis CLI参数
private string BuildRedisCliArguments(string command)
{
var args = new StringBuilder($"-h {_settings.Host} -p {_settings.Port}");
if (!string.IsNullOrEmpty(_settings.Auth))
{
args.Append($" -a {_settings.Auth}");
}
args.Append($" {command}");
return args.ToString();
}
// 辅助方法:异步读取流
private async Task ReadStreamAsync(StreamReader reader, StringBuilder builder)
{
char[] buffer = new char[4096];
int bytesRead;
while ((bytesRead = await reader.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
builder.Append(buffer, 0, bytesRead);
}
}
private string GetRedisCliPath()
{
// Try configured path first
if (!string.IsNullOrEmpty(_settings.RedisCliPath) && File.Exists(_settings.RedisCliPath))
{
return _settings.RedisCliPath;
}
// Try current directory
var currentDirPath = Path.Combine(Directory.GetCurrentDirectory(), "redis-cli.exe");
if (File.Exists(currentDirPath))
{
return currentDirPath;
}
// Try PATH environment variable
return "redis-cli.exe"; // Let the system find it
}
private async Task CreateEmergencyBackupAsync()
{
try
{
if (!_settings.EnableAutoBackup)
return;
Log.Information("💾 Creating emergency backup before restart...");
// 确保备份目录存在
if (!Directory.Exists(_settings.BackupDirectory))
{
Directory.CreateDirectory(_settings.BackupDirectory);
}
// 尝试触发BGSAVE如果Redis还在运行
if (IsRedisRunning())
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var saveResult = await ExecuteRedisCliCommandAsync("BGSAVE", cts.Token);
if (!string.IsNullOrEmpty(saveResult) && !saveResult.Contains("ERR"))
{
Log.Information("✅ Background save triggered successfully");
await Task.Delay(2000); // 等待保存完成
}
}
// 复制现有的数据文件
var dataFiles = new[] { "dump.rdb", "appendonly.aof" };
var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
foreach (var dataFile in dataFiles)
{
var sourcePath = Path.Combine(_settings.DataDirectory, dataFile);
if (File.Exists(sourcePath))
{
var backupPath = Path.Combine(_settings.BackupDirectory, $"emergency_{dataFile}_{timestamp}");
File.Copy(sourcePath, backupPath, true);
Log.Information("📁 Backed up {DataFile} to {BackupPath}", dataFile, backupPath);
}
}
}
catch (Exception ex)
{
Log.Warning("⚠️ Failed to create emergency backup: {Error}", ex.Message);
}
}
private async Task ValidateDataFilesAsync()
{
try
{
Log.Information("🔍 Validating Redis data files...");
var dumpFile = Path.Combine(_settings.DataDirectory, "dump.rdb");
var aofFile = Path.Combine(_settings.DataDirectory, "appendonly.aof");
// 检查RDB文件
if (File.Exists(dumpFile))
{
var fileInfo = new FileInfo(dumpFile);
if (fileInfo.Length == 0)
{
Log.Warning("⚠️ RDB file is empty, may be corrupted");
}
else
{
Log.Information("✅ RDB file exists and has data ({Size} bytes)", fileInfo.Length);
}
}
// 检查AOF文件
if (File.Exists(aofFile))
{
var fileInfo = new FileInfo(aofFile);
Log.Information("✅ AOF file exists ({Size} bytes)", fileInfo.Length);
}
// 检查并修复可能的权限问题
await EnsureDataDirectoryPermissionsAsync();
}
catch (Exception ex)
{
Log.Warning("⚠️ Error validating data files: {Error}", ex.Message);
}
}
private Task EnsureDataDirectoryPermissionsAsync()
{
try
{
if (!Directory.Exists(_settings.DataDirectory))
{
Directory.CreateDirectory(_settings.DataDirectory);
Log.Information("📁 Created data directory: {DataDirectory}", _settings.DataDirectory);
}
// 确保目录可写
var testFile = Path.Combine(_settings.DataDirectory, "test_write.tmp");
File.WriteAllText(testFile, "test");
File.Delete(testFile);
Log.Debug("✅ Data directory permissions verified");
}
catch (Exception ex)
{
Log.Error("❌ Data directory permission issue: {Error}", ex.Message);
}
return Task.CompletedTask;
}
private async Task<bool> VerifyRedisHealthAfterRestartAsync()
{
try
{
Log.Information("🏥 Verifying Redis health after restart...");
// 等待Redis完全启动
for (int i = 0; i < 10; i++)
{
if (IsRedisRunning())
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
// 测试基本连接
var pingResult = await ExecuteRedisCliCommandAsync("PING", cts.Token);
if (!string.IsNullOrEmpty(pingResult) && pingResult.Contains("PONG"))
{
Log.Information("✅ Redis is responding to PING");
// 测试基本操作
var setResult = await ExecuteRedisCliCommandAsync("SET health_check_key test_value");
var getResult = await ExecuteRedisCliCommandAsync("GET health_check_key");
var delResult = await ExecuteRedisCliCommandAsync("DEL health_check_key");
if (!string.IsNullOrEmpty(getResult) && getResult.Contains("test_value"))
{
Log.Information("✅ Redis basic operations working correctly");
return true;
}
}
}
await Task.Delay(1000);
}
Log.Warning("⚠️ Redis health verification failed after restart");
return false;
}
catch (Exception ex)
{
Log.Error("❌ Error verifying Redis health: {Error}", ex.Message);
return false;
}
}
public async Task<bool> RecoverFromBackupAsync(string backupPath = null)
{
try
{
Log.Information("🔄 Starting Redis data recovery...");
// 停止Redis
await StopRedisAsync(graceful: false);
// 如果没有指定备份路径,寻找最新的备份
if (string.IsNullOrEmpty(backupPath))
{
backupPath = FindLatestBackup();
}
if (string.IsNullOrEmpty(backupPath) || !File.Exists(backupPath))
{
Log.Error("❌ No valid backup found for recovery");
return false;
}
Log.Information("📁 Recovering from backup: {BackupPath}", backupPath);
// 备份当前损坏的文件
var corruptedBackupDir = Path.Combine(_settings.BackupDirectory, $"corrupted_{DateTime.Now:yyyyMMdd_HHmmss}");
Directory.CreateDirectory(corruptedBackupDir);
var currentDumpFile = Path.Combine(_settings.DataDirectory, "dump.rdb");
if (File.Exists(currentDumpFile))
{
File.Move(currentDumpFile, Path.Combine(corruptedBackupDir, "dump.rdb"));
}
// 恢复备份文件
File.Copy(backupPath, currentDumpFile, true);
Log.Information("✅ Backup file restored");
// 重启Redis
var startResult = await StartRedisAsync();
if (startResult)
{
Log.Information("✅ Redis recovery completed successfully");
return true;
}
else
{
Log.Error("❌ Failed to start Redis after recovery");
return false;
}
}
catch (Exception ex)
{
Log.Error("❌ Error during Redis recovery: {Error}", ex.Message);
return false;
}
}
private string FindLatestBackup()
{
try
{
if (!Directory.Exists(_settings.BackupDirectory))
return null;
var backupFiles = Directory.GetFiles(_settings.BackupDirectory, "*.rdb")
.OrderByDescending(f => new FileInfo(f).CreationTime)
.ToArray();
return backupFiles.FirstOrDefault();
}
catch
{
return null;
}
}
public void Dispose()
{
_redisProcess?.Dispose();
}
}
}