Feature Release Push

Qolors · Oct 25, 2023 · 028df9b · 028df9b
1 parent 6f3e0aa
commit 028df9b
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 45 deletions.
diff --git a/FeedCord/docs/reference.md b/FeedCord/docs/reference.md
@@ -7,8 +7,9 @@
 ### Property References
 
 - **RssUrls**: The list of RSS Feeds you want to get posts from. You need **at least 1 Url** here.
-- **DiscordWebhookUrl**: The created Webhook from your designated Discord Text Channel. *Not Optional*
+- **DiscordWebhookUrl**: The created Webhook from your designated Discord Text Channel.
 - **RssCheckIntervalMinutes**: How often you want to check for new Posts from all of your Url feeds in minutes.
+- **EnableAutoRemove**: If set to true - Feedcord will kick a Url out of the list after 3 failed attempts to parse the content.
 - **AvatarUrl**: The displayed icon for the bot.
 - **AuthorIcon**: The icon displayed for the Author.
 - **AuthorName**: Display name of Author.

diff --git a/FeedCord/src/Common/Config.cs b/FeedCord/src/Common/Config.cs
@@ -14,6 +14,7 @@ internal class Config
         public string FooterImage { get; }
         public int Color { get; }
         public int RssCheckIntervalMinutes { get; }
+        public bool EnableAutoRemove { get; }
 
         public Config(
             string[] urls,
@@ -27,10 +28,11 @@ public Config(
             string fallbackImage,
             string footerImage,
             int color,
-            int rssCheckIntervalMinutes)
+            int rssCheckIntervalMinutes,
+            bool enableAutoRemove)
         {
             Urls = urls ?? throw new ArgumentNullException(nameof(urls));
-            YoutubeUrls = youtubeurls ?? new string[0];
+            YoutubeUrls = youtubeurls ?? throw new ArgumentNullException(nameof(youtubeurls));
             Webhook = webhook ?? throw new ArgumentNullException(nameof(webhook));
             Username = username;
             AvatarUrl = avatarUrl;
@@ -41,6 +43,7 @@ public Config(
             FooterImage = footerImage;
             Color = color;
             RssCheckIntervalMinutes = rssCheckIntervalMinutes;
+            EnableAutoRemove = enableAutoRemove;
         }
     }
 }
diff --git a/FeedCord/src/DiscordNotifier/Notifier.cs b/FeedCord/src/DiscordNotifier/Notifier.cs
@@ -34,7 +34,7 @@ public async Task SendNotificationsAsync(List<Post> newPosts)
 
                 if (response.StatusCode == HttpStatusCode.NoContent)
                 {
-                    logger.LogInformation("[{CurrentTime}]: Response - Successful: Posted new content to Discord Text Channel at {CurrentTime}", DateTime.Now);
+                    logger.LogInformation("[{CurrentTime}]: Response - Successful: Posted new content to Discord Text Channel at {CurrentTime}", DateTime.Now, DateTime.Now);
                 }
                 else
                 {

diff --git a/FeedCord/src/RssReader/FeedProcessor.cs b/FeedCord/src/RssReader/FeedProcessor.cs
@@ -17,6 +17,7 @@ internal class FeedProcessor : IFeedProcessor
         private readonly ILogger<FeedProcessor> logger;
         private readonly Dictionary<string, DateTime> rssFeedData = new();
         private readonly Dictionary<string, DateTime> youtubeFeedData = new();
+        private readonly Dictionary<string, int> rssFeedErrorTracker = new();
 
         private FeedProcessor(
             Config config,
@@ -63,7 +64,7 @@ private async Task InitializeUrlsAsync()
 
             int successCount = rssCount + youtubeCount;
 
-            logger.LogInformation("Set initial datetime for {UrlCount} out of {TotalUrls} on first run", successCount, totalUrls);
+            logger.LogInformation("Tested successfully for {UrlCount} out of {TotalUrls} Urls in Configuration File", successCount, totalUrls);
         }
 
         private async Task<int> GetSuccessCount(string[] urls, bool isYoutube)
@@ -86,6 +87,8 @@ private async Task<int> GetSuccessCount(string[] urls, bool isYoutube)
                     {
                         rssFeedData[url] = DateTime.Now;
                     }
+
+                    rssFeedErrorTracker[url] = 0;
                     successCount++;
                     logger.LogInformation("Successfully initialized URL: {Url}", url);
                 }
@@ -114,26 +117,63 @@ private async Task<bool> TestUrlAsync(string url)
 
         private async Task CheckAndAddNewPostAsync(KeyValuePair<string, DateTime> rssFeed, ConcurrentBag<Post> newPosts, bool isYoutube)
         {
+            logger.LogInformation("Checking if any new posts for {RssFeedKey}...", rssFeed.Key);
 
             var post = await CheckFeedForUpdatesAsync(rssFeed.Key, isYoutube);
 
-            if (post != null && post.PublishDate > rssFeed.Value)
+            if (post is null)
             {
-                if (isYoutube)
-                {
-                    youtubeFeedData[rssFeed.Key] = post.PublishDate;
-                }
-                else
+                rssFeedErrorTracker[rssFeed.Key]++;
+                logger.LogWarning("Failed to fetch or process the RSS feed from {Url}.. Error Count: {ErrorCount}", rssFeed.Key, rssFeedErrorTracker[rssFeed.Key]);
+
+                if (rssFeedErrorTracker[rssFeed.Key] >= 3)
                 {
-                    rssFeedData[rssFeed.Key] = post.PublishDate;
+                    if (config.EnableAutoRemove)
+                    {
+                        logger.LogWarning("Removing Url: {Url} from the list of RSS feeds due to too many errors", rssFeed.Key);
+
+                        if (youtubeFeedData.ContainsKey(rssFeed.Key))
+                        {
+                            youtubeFeedData.Remove(rssFeed.Key);
+                        }
+                        else if (rssFeedData.ContainsKey(rssFeed.Key))
+                        {
+                            rssFeedData.Remove(rssFeed.Key);
+                        }
+
+                        rssFeedErrorTracker.Remove(rssFeed.Key);
+                    }
+                    else
+                    {
+                        logger.LogWarning("Recommend enabling Auto Remove or manually removing the url from the config file");
+                    }
+
+                    return;
                 }
 
-                newPosts.Add(post);
-                logger.LogInformation("Found new post for Url: {RssFeedKey} at {CurrentTime}", rssFeed.Key, DateTime.Now);
+                return;
             }
+
+            if (post.PublishDate <= rssFeed.Value)
+            {
+                return;
+            }
+
+            if (isYoutube)
+            {
+                youtubeFeedData[rssFeed.Key] = post.PublishDate;
+            }
+            else
+            {
+                rssFeedData[rssFeed.Key] = post.PublishDate;
+            }
+
+            newPosts.Add(post);
+            logger.LogInformation("Found new post for Url: {RssFeedKey}", rssFeed.Key);
         }
 
 
+
         private async Task<Post?> CheckFeedForUpdatesAsync(string url, bool isYoutube)
         {
             try

diff --git a/FeedCord/src/Services/DiscordPayloadService.cs b/FeedCord/src/Services/DiscordPayloadService.cs
@@ -19,7 +19,6 @@ public static StringContent BuildPayloadWithPost(Post post)
             {
                 username = _config.Username,
                 avatar_url = _config.AvatarUrl,
-                color = _config.Color,
                 embeds = new[]
                 {
                     new
@@ -42,6 +41,7 @@ public static StringContent BuildPayloadWithPost(Post post)
                             text = $"{post.Tag} - {post.PublishDate:MM/dd/yyyy h:mm tt}",
                             icon_url = _config.FooterImage
                         },
+                        color = _config.Color,
                     }
                 }
             };

diff --git a/FeedCord/src/Services/RssCheckerBackgroundService.cs b/FeedCord/src/Services/RssCheckerBackgroundService.cs
@@ -33,32 +33,25 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
             while (!stoppingToken.IsCancellationRequested)
             {
-                logger.LogInformation("Starting Background Process at {CurrentTime}", DateTime.Now);
+                logger.LogInformation("Starting Background Process at {CurrentTime}..", DateTime.Now);
                 await RunRoutineBackgroundProcessAsync();
-                logger.LogInformation("Finished Background Process at {CurrentTime}", DateTime.Now);
+                logger.LogInformation("Finished Background Process at {CurrentTime}..", DateTime.Now);
                 await Task.Delay(TimeSpan.FromMinutes(delayTime), stoppingToken);
             }
         }
 
         private async Task RunRoutineBackgroundProcessAsync()
         {
-            try
-            {
-                var posts = await feedProcessor.CheckForNewPostsAsync();
+            var posts = await feedProcessor.CheckForNewPostsAsync();
 
-                if (posts.Count > 0)
-                {
-                    logger.LogInformation("Found {PostCount} new posts at {CurrentTime}", posts.Count, DateTime.Now);
-                    await notifier.SendNotificationsAsync(posts);
-                }
-                else
-                {
-                    logger.LogInformation("Found no new posts at {CurrentTime}. Ending background process.", DateTime.Now);
-                }
+            if (posts.Count > 0)
+            {
+                logger.LogInformation("Found {PostCount} new posts..", posts.Count);
+                await notifier.SendNotificationsAsync(posts);
             }
-            catch (Exception ex)
+            else
             {
-                logger.LogWarning(ex, "A warning occurred while checking for new posts at {CurrentTime}.", DateTime.Now);
+                logger.LogInformation("Found no new posts. Ending background process..");
             }
         }
     }

diff --git a/FeedCord/src/Services/RssProcessorService.cs b/FeedCord/src/Services/RssProcessorService.cs
@@ -32,7 +32,7 @@ public RssProcessorService(
 
                 if (latestPost == null)
                 {
-                    logger.LogError("No items found in the RSS feed. Is this a traditional RSS Feed?");
+                    logger.LogError("No items found in the RSS feed. FeedCord only supports Traditional RSS Feeds");
                     return null;
                 }
 
@@ -51,7 +51,6 @@ public RssProcessorService(
             }
             catch (Exception ex)
             {
-                logger.LogError(ex, "Error parsing the RSS feed.");
                 return null;
             }
         }

diff --git a/FeedCord/src/Services/YoutubeParsingService.cs b/FeedCord/src/Services/YoutubeParsingService.cs
@@ -26,13 +26,8 @@ public YoutubeParsingService(IHttpClientFactory httpClientFactory, ILogger<Youtu
             if (node != null)
             {
                 var hrefValue = node.GetAttributeValue("href", "");
-                logger.LogCritical("Load Href: {HrefValue}", hrefValue);
                 return await GetRecentPost(hrefValue);
             }
-            else
-            {
-                logger.LogError("Failed to retreive Youtube XML Url");
-            }
 
             return null;
         }
@@ -69,7 +64,6 @@ public YoutubeParsingService(IHttpClientFactory httpClientFactory, ILogger<Youtu
             }
             catch(Exception ex)
             {
-                logger.LogError(ex, "Failed to get Youtube RSS Feed");
                 return null;
             }
         }

diff --git a/FeedCord/src/Startup.cs b/FeedCord/src/Startup.cs
@@ -35,7 +35,7 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
                     var config = hostContext.Configuration;
 
                     var rssUrls = config.GetSection("RssUrls").Get<string[]>();
-                    var youtubeUrls = config.GetValue<string[]>("YoutubeUrls");
+                    var youtubeUrls = config.GetSection("YoutubeUrls").Get<string[]>();
                     var discordWebhookUrl = config.GetValue<string>("DiscordWebhookUrl");
                     var username = config.GetValue<string>("Username");
                     var avatarUrl = config.GetValue<string>("AvatarUrl");
@@ -46,6 +46,7 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
                     var footerImage = config.GetValue<string>("FooterImage");
                     var color = config.GetValue<int>("Color");
                     var rssCheckIntervalMinutes = config.GetValue<int>("RssCheckIntervalMinutes");
+                    var enableAutoRemove = config.GetValue<bool>("EnableAutoRemove");
 
                     var appConfig = new Config(
                         rssUrls,
@@ -59,12 +60,15 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
                         fallbackImage,
                         footerImage,
                         color,
-                        rssCheckIntervalMinutes);
+                        rssCheckIntervalMinutes,
+                        enableAutoRemove);
+
                     services.AddHttpClient("Default", httpClient => 
                         {
                             httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
                             httpClient.Timeout.Add(TimeSpan.FromSeconds(30));
                         });
+
                     services
                         .AddScoped<IRssProcessorService, RssProcessorService>()
                         .AddTransient<IOpenGraphService, OpenGraphService>()

diff --git a/README.md b/README.md
@@ -37,22 +37,33 @@ FeedCord is a dead-simple RSS Reader designed to integrate seamlessly with Disco
     "RSS URLS",
     "HERE"
   ],
-  "DiscordWebhookUrl": "YOUR_WEBHOOK_URL_HERE",
-  "RssCheckIntervalMinutes": 10,
+  "YoutubeUrls": [
+    "YOUR",
+    "YOUTUBE CHANNEL URLS",
+    "HERE",
+    "eg. https://www.youtube.com/@IGN"
+  ],
+  "DiscordWebhookUrl": "https://discordapp.com/api/webhooks/1139357708546478200/ncB3dshJOPkQhthwOFQibeNt6YI-1_DiFbg0B3ZecfxchnbCGQNdG-m3PxqDdDSvt5Kk",
+  "RssCheckIntervalMinutes": 3,
+  "EnableAutoRemove": true,
   "Username": "FeedCord",
   "AvatarUrl": "https://i.imgur.com/1asmEAA.png",
   "AuthorIcon": "https://i.imgur.com/1asmEAA.png",
   "AuthorName": "FeedCord",
   "AuthorUrl": "https://github.com/Qolors/FeedCord",
   "FallbackImage": "https://i.imgur.com/f8M2Y5s.png",
   "FooterImage": "https://i.imgur.com/f8M2Y5s.png",
-  "Color": 16744576
+  "Color": 8411391
 }
 ```
 
 Make sure to replace placeholders (e.g., `YOUR RSS URLS HERE`, `YOUR_WEBHOOK_URL_HERE`) with your actual data.
 You can see what each property does [here](https://github.com/Qolors/FeedCord/blob/master/FeedCord/docs/reference.md).
 
+**Note:**: For a Youtube Channel FeedCord handles retreiving the RSS/XML Url for you. This means you just need to provide the Youtube Channel's home page url. This will look something like:
+- 'https://www.youtube.com/@APopularChannel'
+- 'https://www.youtube.com/channel/UCVfiai2'
+
 ---
 
 ### 3. Docker Deployment
@@ -74,7 +85,7 @@ services:
 
 Replace `./PATH/TO/MY/JSON/FILE/` with the actual path to your `appsettings.json`.
 
-**Note:**: Depending on your architecture, use `qolors/feedcord:latest` for amd64 architecture, or `qolors/feedcord:latest-arm64` for arm64 architecture. Ensure to uncomment the appropriate line in the docker-compose.yml as per your system's architecture. If you need a different please open a request.
+**Note:** Depending on your architecture, use `qolors/feedcord:latest` for amd64 architecture, or `qolors/feedcord:latest-arm64` for arm64 architecture. Ensure to uncomment the appropriate line in the docker-compose.yml as per your system's architecture. If you need a different please open a request.
 
 **Step 2:** Navigate to your `FeedCord` directory in your terminal and run:
 
@@ -108,6 +119,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+<details>
+  <summary>[1.2.0] - 2023-10-25</summary>
+  
+  ### Added
+  - Added Support for Youtube Channel Feeds in configuration file.
+  - Added an optional Auto Remove option in configuration file for bad URL Feeds to get booted out of the list after multiple failed attempts.
+
+  ### Changed
+  - Improved container logging messages for better readability.
+
+  ### Fixed
+  - Color setting in configuration now properly works for the embed message
+  - Fixed the handling of errors and removed from logging to reduce spam.
+  - Fixed a known logging index error.
+
+</details>
+
 <details>
   <summary>[1.1.0] - 2023-10-16</summary>
-Original file line number
+Diff line change
@@ Expand Up / @@ -34,7 +34,7 @@ public async Task SendNotificationsAsync(List<Post> newPosts) @@
                     if (response.StatusCode == HttpStatusCode.NoContent)
                     {
-                        logger.LogInformation("[{CurrentTime}]: Response - Successful: Posted new content to Discord Text Channel at {CurrentTime}", DateTime.Now);
+                        logger.LogInformation("[{CurrentTime}]: Response - Successful: Posted new content to Discord Text Channel at {CurrentTime}", DateTime.Now, DateTime.Now);
                     }
                     else
                     {
@@ Expand Down @@