using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using PhoenixLib.ServiceBus; using WingsAPI.Communication; using WingsAPI.Communication.Services; using WingsAPI.Communication.Services.Messages; using WingsAPI.Communication.Services.Requests; using WingsAPI.Communication.Services.Responses; using WingsEmu.Health; namespace Master.Services.Maintenance { public class GrpcClusterStatusService : IClusterStatusService { private readonly IMessagePublisher _activateMaintenancePublisher; private readonly IMessagePublisher _deactivateMaintenancePublisher; private readonly IStatusManager _maintenanceManager; private readonly IMessagePublisher _maintenanceNotificationPublisher; private readonly TimeSpan[] _scheduledShutdownMessages = { TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(30), TimeSpan.FromHours(1), TimeSpan.FromHours(2), TimeSpan.FromHours(4), TimeSpan.FromHours(8), TimeSpan.FromHours(16), TimeSpan.FromDays(1) }; private readonly IMessagePublisher _serviceFlushAllPublisher; private readonly IMessagePublisher _serviceKickAllPublisher; private bool _maintenanceApplied; private CancellationTokenSource _scheduledShutdownTokenSource; public GrpcClusterStatusService(IStatusManager maintenanceManager, IMessagePublisher activateMaintenancePublisher, IMessagePublisher deactivateMaintenancePublisher, IMessagePublisher maintenanceNotificationPublisher, IMessagePublisher serviceFlushAllPublisher, IMessagePublisher serviceKickAllPublisher) { _maintenanceManager = maintenanceManager; _activateMaintenancePublisher = activateMaintenancePublisher; _deactivateMaintenancePublisher = deactivateMaintenancePublisher; _maintenanceNotificationPublisher = maintenanceNotificationPublisher; _serviceFlushAllPublisher = serviceFlushAllPublisher; _serviceKickAllPublisher = serviceKickAllPublisher; } public Task GetAllServicesStatus(EmptyRpcRequest req) { IReadOnlyList services = _maintenanceManager.GetAllServicesStatus(); IEnumerable tmp = services.Select(s => new Service { Id = s.ServiceName, Status = (ServiceHealthStatus)s.Status, LastUpdate = s.LastUpdate }); return Task.FromResult(new ServiceGetAllResponse { Services = tmp.ToList() }); } public async Task GetServiceStatusByNameAsync(ServiceBasicRequest req) { ServiceStatus service = _maintenanceManager.GetServiceByName(req.ServiceName); if (service == null) { return new ServiceGetStatusByNameResponse { ResponseType = RpcResponseType.GENERIC_SERVER_ERROR }; } return new ServiceGetStatusByNameResponse { ResponseType = RpcResponseType.SUCCESS, Service = new Service { Id = service.ServiceName, Status = (ServiceHealthStatus)service.Status, LastUpdate = service.LastUpdate } }; } public async Task EnableMaintenanceMode(ServiceBasicRequest req) { string serviceName = req.ServiceName; ServiceStatus serviceStatus = _maintenanceManager.GetServiceByName(serviceName); if (serviceStatus.Status != ServiceStatusType.ONLINE) { return new BasicRpcResponse { ResponseType = RpcResponseType.GENERIC_SERVER_ERROR }; } await _activateMaintenancePublisher.PublishAsync(new ServiceMaintenanceActivateMessage { TargetServiceName = serviceName }); return new BasicRpcResponse { ResponseType = RpcResponseType.SUCCESS }; } public async Task DisableMaintenanceMode(ServiceBasicRequest req) { string serviceName = req.ServiceName; ServiceStatus serviceStatus = _maintenanceManager.GetServiceByName(serviceName); if (serviceStatus.Status != ServiceStatusType.UNDER_MAINTENANCE) { return new BasicRpcResponse { ResponseType = RpcResponseType.GENERIC_SERVER_ERROR }; } await _deactivateMaintenancePublisher.PublishAsync(new ServiceMaintenanceDeactivateMessage { TargetServiceName = serviceName }); return new BasicRpcResponse { ResponseType = RpcResponseType.SUCCESS }; } public async Task ScheduleGeneralMaintenance(ServiceScheduleGeneralMaintenanceRequest maintenanceRequest) { if (_maintenanceApplied) { return new BasicRpcResponse { ResponseType = RpcResponseType.MAINTENANCE_MODE }; } bool rescheduled = _scheduledShutdownTokenSource != null; if (rescheduled) { _scheduledShutdownTokenSource.Cancel(); _scheduledShutdownTokenSource.Dispose(); } _scheduledShutdownTokenSource = new CancellationTokenSource(); Task.Run(() => ScheduleGeneralMaintenance(maintenanceRequest.ShutdownTimeSpan, _scheduledShutdownTokenSource.Token)); await _maintenanceNotificationPublisher.PublishAsync(new ServiceMaintenanceNotificationMessage { NotificationType = rescheduled ? ServiceMaintenanceNotificationType.Rescheduled : ServiceMaintenanceNotificationType.ScheduleWarning, Reason = maintenanceRequest.Reason, TimeLeft = maintenanceRequest.ShutdownTimeSpan }, _scheduledShutdownTokenSource.Token); return new BasicRpcResponse { ResponseType = RpcResponseType.SUCCESS }; } public async Task UnscheduleGeneralMaintenance(EmptyRpcRequest emptyRpcRequest) { if (_maintenanceApplied) { return new BasicRpcResponse { ResponseType = RpcResponseType.MAINTENANCE_MODE }; } if (_scheduledShutdownTokenSource == null) { return new BasicRpcResponse { ResponseType = RpcResponseType.GENERIC_SERVER_ERROR }; } _scheduledShutdownTokenSource.Cancel(); _scheduledShutdownTokenSource.Dispose(); _scheduledShutdownTokenSource = null; await _maintenanceNotificationPublisher.PublishAsync(new ServiceMaintenanceNotificationMessage { NotificationType = ServiceMaintenanceNotificationType.ScheduleStopped }); return new BasicRpcResponse { ResponseType = RpcResponseType.SUCCESS }; } public async Task ExecuteGeneralEmergencyMaintenance(ServiceExecuteGeneralEmergencyMaintenanceRequest shutdownRequest) { if (_maintenanceApplied) { return new BasicRpcResponse { ResponseType = RpcResponseType.MAINTENANCE_MODE }; } if (_scheduledShutdownTokenSource != null) { _scheduledShutdownTokenSource.Cancel(); _scheduledShutdownTokenSource.Dispose(); _scheduledShutdownTokenSource = null; } await ExecuteGeneralMaintenance(true, shutdownRequest.Reason); return new BasicRpcResponse { ResponseType = RpcResponseType.SUCCESS }; } public async Task LiftGeneralMaintenance(EmptyRpcRequest shutdownRequest) { if (!_maintenanceApplied) { return new BasicRpcResponse { ResponseType = RpcResponseType.MAINTENANCE_MODE }; } await LiftGeneralMaintenance(); return new BasicRpcResponse { ResponseType = RpcResponseType.SUCCESS }; } private async Task ScheduleGeneralMaintenance(TimeSpan shutdownTimeSpan, CancellationToken cancellationToken) { DateTime shutdownDateTime = DateTime.UtcNow + shutdownTimeSpan; int i; //Remove non needed messages for (i = _scheduledShutdownMessages.Length - 1; i >= 0; i--) { if (shutdownTimeSpan <= _scheduledShutdownMessages[i]) { continue; } break; } //Send notification messages until we run out of them while (i >= 0) { TimeSpan timeLeft = shutdownDateTime - DateTime.UtcNow; TimeSpan message = _scheduledShutdownMessages[i]; if (timeLeft <= message) { await _maintenanceNotificationPublisher.PublishAsync(new ServiceMaintenanceNotificationMessage { NotificationType = ServiceMaintenanceNotificationType.ScheduleWarning, TimeLeft = message }, cancellationToken); i--; continue; } await Task.Delay(500, cancellationToken); } if (cancellationToken.IsCancellationRequested) { return; } //Start shutdown await ExecuteGeneralMaintenance(false); } private async Task ExecuteGeneralMaintenance(bool emergency, string reason = null) { _maintenanceApplied = true; await _activateMaintenancePublisher.PublishAsync(new ServiceMaintenanceActivateMessage { IsGlobal = true }); await _serviceKickAllPublisher.PublishAsync(new ServiceKickAllMessage { IsGlobal = true }); await Task.Delay(5000); await _serviceFlushAllPublisher.PublishAsync(new ServiceFlushAllMessage { IsGlobal = true }); await _maintenanceNotificationPublisher.PublishAsync(new ServiceMaintenanceNotificationMessage { NotificationType = emergency ? ServiceMaintenanceNotificationType.EmergencyExecuted : ServiceMaintenanceNotificationType.Executed, Reason = reason }); } private async Task LiftGeneralMaintenance() { await _deactivateMaintenancePublisher.PublishAsync(new ServiceMaintenanceDeactivateMessage { IsGlobal = true }); await _maintenanceNotificationPublisher.PublishAsync(new ServiceMaintenanceNotificationMessage { NotificationType = ServiceMaintenanceNotificationType.Lifted }); _maintenanceApplied = false; } } }