+ Added new setting ZkbUrl. Show ZKB button on web UI if set.

+ Added new settings `AuxiliaryDiscordGuildIds` to Web Auth. If specified the auth checks will also be applied to these guilds allowing auth to somewhat work on several guilds at once.
+ Modified Discord welcome messages (if enabled) to be printed in aux guilds default channels
+ `UseSocketsForZKillboard` setting has been deprecated. ZKill module will now always use websockets by default.
+ Improved ZKill module to cache KMs while there's an outage of ESI connection
+ Improved module logic for modules which do not require ESI to run while ESI is not available (Timer notifications)
+ Fixed some connection outage exceptions

ThunderED.Shared/Settings/Settings.cs

@@ -235,9 +235,9 @@ public class IndustryJobFilter
 
     public class ZKBSettingsModuleSettings
     {
-        [Comment("Optional ZKill RedisQ queue name to fetch kills from. Could be any text value but make sure it is not simple and is quite unique")]
-        public string ZkillLiveFeedRedisqID { get; set; }
-        public bool UseSocketsForZKillboard { get; set; } = true;
+        //[Comment("Optional ZKill RedisQ queue name to fetch kills from. Could be any text value but make sure it is not simple and is quite unique")]
+        //public string ZkillLiveFeedRedisqID { get; set; }
+       // public bool UseSocketsForZKillboard { get; set; } = true;
         public string ZKillboardWebSocketUrl { get; set; } = "wss://zkillboard.com/websocket/";
         [Comment("Try avoid duplicate killmails across all radius and live kill feeds")]
         public bool AvoidDupesAcrossAllFeeds { get; set; } = false;

ThunderED/API/ESIAPI.cs

@@ -457,7 +457,7 @@ public async Task<bool> IsServerOnline(string reason)
         {
             var status = await GetServerStatus(reason);
             if (status?.Data == null || status.Data.IsNoConnection) return false;
-            return status.Result.players > 20;
+            return (status.Result?.players ?? 0) > 20;
         }
 
         public async Task<int> IsServerOnlineEx(string reason)
@@ -472,7 +472,7 @@ public async Task<int> IsServerOnlineEx(string reason)
                     return -1; //esi down
                 }
 
-                return res.Result.players > 20 ? 1 : 0;
+                return (res.Result?.players ?? 0) > 20 ? 1 : 0;
             }
             catch
             {

ThunderED/API/ZKillAPI.cs

@@ -42,18 +42,16 @@ public partial class ZKillAPI: CacheBase, IDisposable
                         _webSocket = null;
                         return null;
                     }
-                    else
+
+                    if (!_webSocket.Send("{\"action\":\"sub\",\"channel\":\"killstream\"}"))
                     {
-                        if (!_webSocket.Send("{\"action\":\"sub\",\"channel\":\"killstream\"}"))
-                        {
-                            _webSocket?.Dispose();
-                            _webSocket = null;
-                            return null;
-                        }
-
-                        await LogHelper.LogInfo("ZKB feed core WebSocket connect successful!", LogCat.ZKill);
+                        _webSocket?.Dispose();
+                        _webSocket = null;
+                        return null;
                     }
 
+                    await LogHelper.LogInfo("ZKB feed core WebSocket connect successful!", LogCat.ZKill);
+
                 }
 
                 if (!_webMailsQueue.IsEmpty && _webMailsQueue.TryDequeue(out var km))
@@ -82,7 +80,7 @@ private async void WebSocket_OnMessage(object sender, string message)
             }
         }
 
-        internal async Task<JsonZKill.ZKillboard> GetRedisqResponce()
+       /* internal async Task<JsonZKill.ZKillboard> GetRedisqResponce()
         {
             var redisqID = SettingsManager.Settings.ZKBSettingsModule.ZkillLiveFeedRedisqID;
 			var request = string.IsNullOrEmpty(redisqID)
@@ -96,7 +94,7 @@ private async void WebSocket_OnMessage(object sender, string message)
             }
 				//await LogHelper.LogError("[GetRedisqResponce] Null data!", LogCat.ZKill, false);
 			return data;
-        }
+        }*/
 
         internal async Task<List<JsonClasses.ESIKill>> GetCharacterKills(object characterId)
         {

ThunderED/Classes/DiscordCommands.cs

@@ -348,7 +348,8 @@ public async Task TQStatus()
             if(IsForbidden()) return;
             var tq = await APIHelper.ESIAPI.GetServerStatus("ESIAPI");
             var isOnline = tq.Result != null && tq.Result.players > 20;
-            await APIHelper.DiscordAPI.ReplyMessageAsync(Context, LM.Get("tqStatusText", isOnline ? LM.Get("online") : LM.Get("offline"), tq?.Result.players ?? 0), true);
+
+            await APIHelper.DiscordAPI.ReplyMessageAsync(Context, LM.Get("tqStatusText", isOnline ? LM.Get("online") : LM.Get("offline"), tq?.Result?.players ?? 0, APIHelper.IsDiscordAvailable, await APIHelper.ESIAPI.IsServerOnline("default")), true);
         }
 
         internal const string CMD_TIMERS= "timers";

ThunderED/Classes/TickManager.cs

@@ -189,8 +189,6 @@ public static async void Tick(object stateInfo)
 
                 await ContinuousCheckModule.OneSec_TQStatusPost(_asyncNow);
 
-                if(IsNoConnection || IsESIUnreachable) return;
-
                 await Modules.ParallelForEachAsync(async module =>
                 {
                     await module.RunInternal(null);

ThunderED/Languages/en-US.json

@@ -186,7 +186,7 @@
   "Location": "Location",
   "Aggressor": "Agressor",
   "backText": "Go Back",
-  "tqStatusText": "```\nTQ status: {0}\nPlayers: {1}\n```",
+  "tqStatusText": "```\nTQ status: {0}\nDiscord online: {2}\nESI online: {3}\nPlayers: {1}\n```",
   "Online": "Online",
   "Offline": "Offline",
   "incursionUpdateHeader": "Incursion status update for {0} const of {1} region",

ThunderED/Languages/ru-RU.json

@@ -177,7 +177,7 @@
   "Location": "Локация",
   "Aggressor": "Агрессор",
   "backText": "Назад",
-  "tqStatusText": "```\nСтатус TQ: {0}\nИгроков: {1}\n```",
+  "tqStatusText": "```\nСтатус TQ: {0}\nDiscord онлайн: {2}\nESI онлайн: {3}\nИгроков: {1}\n```",
   "Online": "Работает",
   "Offline": "Выключен",
   "incursionUpdateHeader": "Статус вторжения в консте {0} региона {1}",

ThunderED/Modules/AuthCheckModule.cs

@@ -19,7 +19,9 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if(IsRunning) return;
-            if(Settings.Config.ModuleAuthWeb && TickManager.GetModule<WebAuthModule>().IsEntityInitFailed) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
+
+            if (Settings.Config.ModuleAuthWeb && TickManager.GetModule<WebAuthModule>().IsEntityInitFailed) return;
             if(!Settings.Config.ModuleAuthCheck) return;
 
             IsRunning = true;

ThunderED/Modules/ContinuousCheckModule.cs

@@ -31,7 +31,9 @@ public class ContinuousCheckModule: AppModuleBase
 
         public override async Task Run(object prm)
         {
-            if(IsRunning || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
+
+            if (IsRunning || !APIHelper.IsDiscordAvailable) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/ContractNotificationsModule.cs

@@ -46,6 +46,7 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if (IsRunning || !Settings.Config.ModuleContractNotifications || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/HRMModule.cs

@@ -2,6 +2,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+
+using ThunderED.Classes;
 using ThunderED.Classes.Enums;
 using ThunderED.Helpers;
 using ThunderED.Thd;
@@ -43,8 +45,9 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if (!Settings.Config.ModuleHRM) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
 
-            if(IsRunning) return;
+            if (IsRunning) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/IncursionNotifyModule.cs

@@ -4,6 +4,8 @@
 using System.Text;
 using System.Threading.Tasks;
 using Discord;
+
+using ThunderED.Classes;
 using ThunderED.Helpers;
 using ThunderED.Json;
 
@@ -24,6 +26,7 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if(IsRunning || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/IndustrialJobsModule.cs

@@ -37,6 +37,7 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if (IsRunning || !Settings.Config.ModuleIndustrialJobs || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/MailModule.cs

@@ -36,6 +36,7 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if (IsRunning || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/MiningScheduleModule.cs

@@ -80,6 +80,7 @@ public async Task<WebQueryResult> ProcessRequest(string query, CallbackTypeEnum
         {
             if (!Settings.Config.ModuleMiningSchedule)
                 return WebQueryResult.False;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return WebQueryResult.EsiFailure;
 
             try
             {

ThunderED/Modules/NotificationModule.cs

@@ -36,6 +36,8 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if(IsRunning || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
+
             IsRunning = true;
             try
             {

ThunderED/Modules/NullCampaignModule.cs

@@ -27,6 +27,7 @@ public override async Task Run(object prm)
         {
 
             if (IsRunning || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
             IsRunning = true;
 
             await ProcessExistingCampaigns();

ThunderED/Modules/SovTrackerModule.cs

@@ -5,6 +5,8 @@
 using System.Text;
 using System.Threading.Tasks;
 using Discord;
+
+using ThunderED.Classes;
 using ThunderED.Helpers;
 using ThunderED.Json;
 
@@ -39,6 +41,7 @@ public override async Task Initialize()
         public override async Task Run(object prm)
         {
             if (IsRunning || !APIHelper.IsDiscordAvailable) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/StructureManagementModule.cs

@@ -55,6 +55,7 @@ public override async Task Run(object prm)
         {
 
             if(!Settings.Config.ModuleStructureManagement || _isRunning) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
             _isRunning = true;
             try
             {

ThunderED/Modules/Sub/ZKillLiveFeedModule.cs

@@ -2,7 +2,11 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
+
+using Dasync.Collections;
+
 using ThunderED.Classes;
 using ThunderED.Helpers;
 using ThunderED.Json.ZKill;
@@ -15,7 +19,8 @@ public class ZKillLiveFeedModule: AppModuleBase, IDisposable
         public static List<Func<JsonZKill.Killmail, Task>> Queryables = new List<Func<JsonZKill.Killmail, Task>>();
         private readonly List<long> _receivedIds = new List<long>();
 
-
+        private ConcurrentQueue<JsonZKill.Killmail> _kmPool = new ConcurrentQueue<JsonZKill.Killmail>();
+
         private static readonly ConcurrentQueue<long> SharedIdPool = new ConcurrentQueue<long>();
         /// <summary>
         /// Update pool with new ID
@@ -34,50 +39,96 @@ internal static bool IsInSharedPool(long id)
             return SharedIdPool.Contains(id);
         }
 
-        public ZKillLiveFeedModule()
+        public override async Task Initialize()
         {
             Settings.ZKBSettingsModule.OldKMDaysThreshold = Settings.ZKBSettingsModule.OldKMDaysThreshold < 0
                 ? 0
                 : Settings.ZKBSettingsModule.OldKMDaysThreshold;
+
+            if (string.IsNullOrEmpty(SettingsManager.Settings.ZKBSettingsModule.ZKillboardWebSocketUrl))
+            {
+                await LogHelper.LogWarning($"ZKB socket param `ZKillboardWebSocketUrl` is not specified! Set it and restart the bot.",
+                    LogCat.ZKill);
+                Settings.Config.ModuleLiveKillFeed = false;
+            }
         }
 
+        //private volatile bool _threadisRunning = false;
+
         public override async Task Run(object prm)
         {
             if (IsRunning || Queryables.Count == 0 || Program.IsClosing || !APIHelper.IsDiscordAvailable) return;
             if(!Settings.Config.ModuleLiveKillFeed) return;
 
+            if (TickManager.IsESIUnreachable) return; //TODO pooling
+
             var minus = Settings.ZKBSettingsModule.OldKMDaysThreshold == 0 ? DateTime.Now : DateTime.Now.Subtract(TimeSpan.FromDays(Settings.ZKBSettingsModule.OldKMDaysThreshold));
             try
             {
                 IsRunning = true;
-                if (TickManager.IsNoConnection || !Queryables.Any()) return;
-                JsonZKill.Killmail entry = null;
+                if (!Queryables.Any()) return;
+               // JsonZKill.Killmail entry = null;
 
-                if (Settings.ZKBSettingsModule.UseSocketsForZKillboard)
+                /*  if (Settings.ZKBSettingsModule.UseSocketsForZKillboard)
+                {*/
+                do
                 {
                     var currentEntry = await APIHelper.ZKillAPI.GetSocketResponce();
-                    if (currentEntry == null) return;
-                    if (!IsUniqueId(currentEntry.killmail_id)) return;
-                    entry = currentEntry;
-                }
+                    if (currentEntry == null) break;
+                    if (!IsUniqueId(currentEntry.killmail_id)) continue;
+                    //do the minus days check
+                    if (currentEntry.killmail_time < minus) continue;
+
+                    _kmPool.Enqueue(currentEntry);
+                    if (TickManager.IsConnected && !TickManager.IsESIUnreachable)
+                        if (_kmPool.Count >= 10)
+                            break;
+                } while (true);
+                /* }
                 else
                 {
                     var currentEntry =  await APIHelper.ZKillAPI.GetRedisqResponce();
                     if (currentEntry?.package == null) return;
                     if (!IsUniqueId(currentEntry.package.killID)) return;
                     currentEntry.package.killmail.zkb = currentEntry.package.zkb;
                     entry = currentEntry.package.killmail;
-                }
+                }*/
 
-                //do the minus days check
-                if(entry.killmail_time < minus)
+                //skip on comms outage
+                if (TickManager.IsESIUnreachable || TickManager.IsNoConnection)
                     return;
 
-                foreach (var q in Queryables)
+                //if(_threadisRunning) return;
+
+                //use awaitable threads if pool is big
+               /* if (_kmPool.Count > 15)
                 {
-                    await q(entry).ConfigureAwait(false);
-                }
+                    _threadisRunning = true;
+
+                    await Queryables.ParallelForEachAsync(async q =>
+                    {
+                        var hasItems = true;
+                        do
+                        {
+                            hasItems = _kmPool.TryDequeue(out var entry);
+                            if (hasItems)
+                                await q(entry);
+                        } while (hasItems);
+                    }, SettingsManager.MaxConcurrentThreads);
 
+                    _threadisRunning = false;
+                }
+                else
+                {*/
+                    var hasItems = true;
+                    do
+                    {
+                        hasItems = _kmPool.TryDequeue(out var entry);
+                        if (!hasItems) continue;
+                        foreach (var q in Queryables)
+                            await q(entry).ConfigureAwait(false);
+                    } while (hasItems);
+               // }
             }
             catch (Exception ex)
             {

ThunderED/Modules/WebAuthModule.cs

@@ -83,8 +83,9 @@ await APIHelper.DiscordAPI.SendMessageAsync(Settings.WebAuthModule.AuthReportCha
         public override async Task Run(object prm)
         {
             if (!Settings.Config.ModuleAuthWeb || IsEntityInitFailed) return;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return;
 
-            if(IsRunning) return;
+            if (IsRunning) return;
             IsRunning = true;
             try
             {

ThunderED/Modules/WebAuthModule_WebPart.cs

@@ -26,6 +26,8 @@ private async Task<WebQueryResult> ProcessAuth(string query, CallbackTypeEnum ca
         {
             if (!Settings.Config.ModuleAuthWeb)
                 return WebQueryResult.False;
+            if (TickManager.IsNoConnection || TickManager.IsESIUnreachable) return WebQueryResult.EsiFailure;
+
             try
             {
                 RunningRequestCount++;