From 028df9b2378ecff5552735ef40932f341aadea6e Mon Sep 17 00:00:00 2001 From: cmick Date: Wed, 25 Oct 2023 14:42:23 -0400 Subject: [PATCH] Feature Release Push --- FeedCord/docs/reference.md | 3 +- FeedCord/src/Common/Config.cs | 7 ++- FeedCord/src/DiscordNotifier/Notifier.cs | 2 +- FeedCord/src/RssReader/FeedProcessor.cs | 60 +++++++++++++++---- .../src/Services/DiscordPayloadService.cs | 2 +- .../Services/RssCheckerBackgroundService.cs | 25 +++----- FeedCord/src/Services/RssProcessorService.cs | 3 +- .../src/Services/YoutubeParsingService.cs | 6 -- FeedCord/src/Startup.cs | 8 ++- README.md | 36 +++++++++-- 10 files changed, 107 insertions(+), 45 deletions(-) diff --git a/FeedCord/docs/reference.md b/FeedCord/docs/reference.md index 29ffe0f..5e76484 100644 --- a/FeedCord/docs/reference.md +++ b/FeedCord/docs/reference.md @@ -7,8 +7,9 @@ ### Property References - **RssUrls**: The list of RSS Feeds you want to get posts from. You need **at least 1 Url** here. -- **DiscordWebhookUrl**: The created Webhook from your designated Discord Text Channel. *Not Optional* +- **DiscordWebhookUrl**: The created Webhook from your designated Discord Text Channel. - **RssCheckIntervalMinutes**: How often you want to check for new Posts from all of your Url feeds in minutes. +- **EnableAutoRemove**: If set to true - Feedcord will kick a Url out of the list after 3 failed attempts to parse the content. - **AvatarUrl**: The displayed icon for the bot. - **AuthorIcon**: The icon displayed for the Author. - **AuthorName**: Display name of Author. diff --git a/FeedCord/src/Common/Config.cs b/FeedCord/src/Common/Config.cs index f3c3e70..373224d 100644 --- a/FeedCord/src/Common/Config.cs +++ b/FeedCord/src/Common/Config.cs @@ -14,6 +14,7 @@ internal class Config public string FooterImage { get; } public int Color { get; } public int RssCheckIntervalMinutes { get; } + public bool EnableAutoRemove { get; } public Config( string[] urls, @@ -27,10 +28,11 @@ public Config( string fallbackImage, string footerImage, int color, - int rssCheckIntervalMinutes) + int rssCheckIntervalMinutes, + bool enableAutoRemove) { Urls = urls ?? throw new ArgumentNullException(nameof(urls)); - YoutubeUrls = youtubeurls ?? new string[0]; + YoutubeUrls = youtubeurls ?? throw new ArgumentNullException(nameof(youtubeurls)); Webhook = webhook ?? throw new ArgumentNullException(nameof(webhook)); Username = username; AvatarUrl = avatarUrl; @@ -41,6 +43,7 @@ public Config( FooterImage = footerImage; Color = color; RssCheckIntervalMinutes = rssCheckIntervalMinutes; + EnableAutoRemove = enableAutoRemove; } } } diff --git a/FeedCord/src/DiscordNotifier/Notifier.cs b/FeedCord/src/DiscordNotifier/Notifier.cs index 9896228..8283b57 100644 --- a/FeedCord/src/DiscordNotifier/Notifier.cs +++ b/FeedCord/src/DiscordNotifier/Notifier.cs @@ -34,7 +34,7 @@ public async Task SendNotificationsAsync(List newPosts) if (response.StatusCode == HttpStatusCode.NoContent) { - logger.LogInformation("[{CurrentTime}]: Response - Successful: Posted new content to Discord Text Channel at {CurrentTime}", DateTime.Now); + logger.LogInformation("[{CurrentTime}]: Response - Successful: Posted new content to Discord Text Channel at {CurrentTime}", DateTime.Now, DateTime.Now); } else { diff --git a/FeedCord/src/RssReader/FeedProcessor.cs b/FeedCord/src/RssReader/FeedProcessor.cs index 6a424b5..fdad41c 100644 --- a/FeedCord/src/RssReader/FeedProcessor.cs +++ b/FeedCord/src/RssReader/FeedProcessor.cs @@ -17,6 +17,7 @@ internal class FeedProcessor : IFeedProcessor private readonly ILogger logger; private readonly Dictionary rssFeedData = new(); private readonly Dictionary youtubeFeedData = new(); + private readonly Dictionary rssFeedErrorTracker = new(); private FeedProcessor( Config config, @@ -63,7 +64,7 @@ private async Task InitializeUrlsAsync() int successCount = rssCount + youtubeCount; - logger.LogInformation("Set initial datetime for {UrlCount} out of {TotalUrls} on first run", successCount, totalUrls); + logger.LogInformation("Tested successfully for {UrlCount} out of {TotalUrls} Urls in Configuration File", successCount, totalUrls); } private async Task GetSuccessCount(string[] urls, bool isYoutube) @@ -86,6 +87,8 @@ private async Task GetSuccessCount(string[] urls, bool isYoutube) { rssFeedData[url] = DateTime.Now; } + + rssFeedErrorTracker[url] = 0; successCount++; logger.LogInformation("Successfully initialized URL: {Url}", url); } @@ -114,26 +117,63 @@ private async Task TestUrlAsync(string url) private async Task CheckAndAddNewPostAsync(KeyValuePair rssFeed, ConcurrentBag newPosts, bool isYoutube) { + logger.LogInformation("Checking if any new posts for {RssFeedKey}...", rssFeed.Key); var post = await CheckFeedForUpdatesAsync(rssFeed.Key, isYoutube); - if (post != null && post.PublishDate > rssFeed.Value) + if (post is null) { - if (isYoutube) - { - youtubeFeedData[rssFeed.Key] = post.PublishDate; - } - else + rssFeedErrorTracker[rssFeed.Key]++; + logger.LogWarning("Failed to fetch or process the RSS feed from {Url}.. Error Count: {ErrorCount}", rssFeed.Key, rssFeedErrorTracker[rssFeed.Key]); + + if (rssFeedErrorTracker[rssFeed.Key] >= 3) { - rssFeedData[rssFeed.Key] = post.PublishDate; + if (config.EnableAutoRemove) + { + logger.LogWarning("Removing Url: {Url} from the list of RSS feeds due to too many errors", rssFeed.Key); + + if (youtubeFeedData.ContainsKey(rssFeed.Key)) + { + youtubeFeedData.Remove(rssFeed.Key); + } + else if (rssFeedData.ContainsKey(rssFeed.Key)) + { + rssFeedData.Remove(rssFeed.Key); + } + + rssFeedErrorTracker.Remove(rssFeed.Key); + } + else + { + logger.LogWarning("Recommend enabling Auto Remove or manually removing the url from the config file"); + } + + return; } - newPosts.Add(post); - logger.LogInformation("Found new post for Url: {RssFeedKey} at {CurrentTime}", rssFeed.Key, DateTime.Now); + return; } + + if (post.PublishDate <= rssFeed.Value) + { + return; + } + + if (isYoutube) + { + youtubeFeedData[rssFeed.Key] = post.PublishDate; + } + else + { + rssFeedData[rssFeed.Key] = post.PublishDate; + } + + newPosts.Add(post); + logger.LogInformation("Found new post for Url: {RssFeedKey}", rssFeed.Key); } + private async Task CheckFeedForUpdatesAsync(string url, bool isYoutube) { try diff --git a/FeedCord/src/Services/DiscordPayloadService.cs b/FeedCord/src/Services/DiscordPayloadService.cs index 2ce6af9..a9b216f 100644 --- a/FeedCord/src/Services/DiscordPayloadService.cs +++ b/FeedCord/src/Services/DiscordPayloadService.cs @@ -19,7 +19,6 @@ public static StringContent BuildPayloadWithPost(Post post) { username = _config.Username, avatar_url = _config.AvatarUrl, - color = _config.Color, embeds = new[] { new @@ -42,6 +41,7 @@ public static StringContent BuildPayloadWithPost(Post post) text = $"{post.Tag} - {post.PublishDate:MM/dd/yyyy h:mm tt}", icon_url = _config.FooterImage }, + color = _config.Color, } } }; diff --git a/FeedCord/src/Services/RssCheckerBackgroundService.cs b/FeedCord/src/Services/RssCheckerBackgroundService.cs index 0e04a75..dfe9fc5 100644 --- a/FeedCord/src/Services/RssCheckerBackgroundService.cs +++ b/FeedCord/src/Services/RssCheckerBackgroundService.cs @@ -33,32 +33,25 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { - logger.LogInformation("Starting Background Process at {CurrentTime}", DateTime.Now); + logger.LogInformation("Starting Background Process at {CurrentTime}..", DateTime.Now); await RunRoutineBackgroundProcessAsync(); - logger.LogInformation("Finished Background Process at {CurrentTime}", DateTime.Now); + logger.LogInformation("Finished Background Process at {CurrentTime}..", DateTime.Now); await Task.Delay(TimeSpan.FromMinutes(delayTime), stoppingToken); } } private async Task RunRoutineBackgroundProcessAsync() { - try - { - var posts = await feedProcessor.CheckForNewPostsAsync(); + var posts = await feedProcessor.CheckForNewPostsAsync(); - if (posts.Count > 0) - { - logger.LogInformation("Found {PostCount} new posts at {CurrentTime}", posts.Count, DateTime.Now); - await notifier.SendNotificationsAsync(posts); - } - else - { - logger.LogInformation("Found no new posts at {CurrentTime}. Ending background process.", DateTime.Now); - } + if (posts.Count > 0) + { + logger.LogInformation("Found {PostCount} new posts..", posts.Count); + await notifier.SendNotificationsAsync(posts); } - catch (Exception ex) + else { - logger.LogWarning(ex, "A warning occurred while checking for new posts at {CurrentTime}.", DateTime.Now); + logger.LogInformation("Found no new posts. Ending background process.."); } } } diff --git a/FeedCord/src/Services/RssProcessorService.cs b/FeedCord/src/Services/RssProcessorService.cs index 00ff320..c0fd436 100644 --- a/FeedCord/src/Services/RssProcessorService.cs +++ b/FeedCord/src/Services/RssProcessorService.cs @@ -32,7 +32,7 @@ public RssProcessorService( if (latestPost == null) { - logger.LogError("No items found in the RSS feed. Is this a traditional RSS Feed?"); + logger.LogError("No items found in the RSS feed. FeedCord only supports Traditional RSS Feeds"); return null; } @@ -51,7 +51,6 @@ public RssProcessorService( } catch (Exception ex) { - logger.LogError(ex, "Error parsing the RSS feed."); return null; } } diff --git a/FeedCord/src/Services/YoutubeParsingService.cs b/FeedCord/src/Services/YoutubeParsingService.cs index f541008..0f72dab 100644 --- a/FeedCord/src/Services/YoutubeParsingService.cs +++ b/FeedCord/src/Services/YoutubeParsingService.cs @@ -26,13 +26,8 @@ public YoutubeParsingService(IHttpClientFactory httpClientFactory, ILogger var config = hostContext.Configuration; var rssUrls = config.GetSection("RssUrls").Get(); - var youtubeUrls = config.GetValue("YoutubeUrls"); + var youtubeUrls = config.GetSection("YoutubeUrls").Get(); var discordWebhookUrl = config.GetValue("DiscordWebhookUrl"); var username = config.GetValue("Username"); var avatarUrl = config.GetValue("AvatarUrl"); @@ -46,6 +46,7 @@ private static IHostBuilder CreateHostBuilder(string[] args) => var footerImage = config.GetValue("FooterImage"); var color = config.GetValue("Color"); var rssCheckIntervalMinutes = config.GetValue("RssCheckIntervalMinutes"); + var enableAutoRemove = config.GetValue("EnableAutoRemove"); var appConfig = new Config( rssUrls, @@ -59,12 +60,15 @@ private static IHostBuilder CreateHostBuilder(string[] args) => fallbackImage, footerImage, color, - rssCheckIntervalMinutes); + rssCheckIntervalMinutes, + enableAutoRemove); + services.AddHttpClient("Default", httpClient => { httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"); httpClient.Timeout.Add(TimeSpan.FromSeconds(30)); }); + services .AddScoped() .AddTransient() diff --git a/README.md b/README.md index ce62e07..dbe5528 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,15 @@ FeedCord is a dead-simple RSS Reader designed to integrate seamlessly with Disco "RSS URLS", "HERE" ], - "DiscordWebhookUrl": "YOUR_WEBHOOK_URL_HERE", - "RssCheckIntervalMinutes": 10, + "YoutubeUrls": [ + "YOUR", + "YOUTUBE CHANNEL URLS", + "HERE", + "eg. https://www.youtube.com/@IGN" + ], + "DiscordWebhookUrl": "https://discordapp.com/api/webhooks/1139357708546478200/ncB3dshJOPkQhthwOFQibeNt6YI-1_DiFbg0B3ZecfxchnbCGQNdG-m3PxqDdDSvt5Kk", + "RssCheckIntervalMinutes": 3, + "EnableAutoRemove": true, "Username": "FeedCord", "AvatarUrl": "https://i.imgur.com/1asmEAA.png", "AuthorIcon": "https://i.imgur.com/1asmEAA.png", @@ -46,13 +53,17 @@ FeedCord is a dead-simple RSS Reader designed to integrate seamlessly with Disco "AuthorUrl": "https://github.com/Qolors/FeedCord", "FallbackImage": "https://i.imgur.com/f8M2Y5s.png", "FooterImage": "https://i.imgur.com/f8M2Y5s.png", - "Color": 16744576 + "Color": 8411391 } ``` Make sure to replace placeholders (e.g., `YOUR RSS URLS HERE`, `YOUR_WEBHOOK_URL_HERE`) with your actual data. You can see what each property does [here](https://github.com/Qolors/FeedCord/blob/master/FeedCord/docs/reference.md). +**Note:**: For a Youtube Channel FeedCord handles retreiving the RSS/XML Url for you. This means you just need to provide the Youtube Channel's home page url. This will look something like: +- 'https://www.youtube.com/@APopularChannel' +- 'https://www.youtube.com/channel/UCVfiai2' + --- ### 3. Docker Deployment @@ -74,7 +85,7 @@ services: Replace `./PATH/TO/MY/JSON/FILE/` with the actual path to your `appsettings.json`. -**Note:**: Depending on your architecture, use `qolors/feedcord:latest` for amd64 architecture, or `qolors/feedcord:latest-arm64` for arm64 architecture. Ensure to uncomment the appropriate line in the docker-compose.yml as per your system's architecture. If you need a different please open a request. +**Note:** Depending on your architecture, use `qolors/feedcord:latest` for amd64 architecture, or `qolors/feedcord:latest-arm64` for arm64 architecture. Ensure to uncomment the appropriate line in the docker-compose.yml as per your system's architecture. If you need a different please open a request. **Step 2:** Navigate to your `FeedCord` directory in your terminal and run: @@ -108,6 +119,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +
+ [1.2.0] - 2023-10-25 + + ### Added + - Added Support for Youtube Channel Feeds in configuration file. + - Added an optional Auto Remove option in configuration file for bad URL Feeds to get booted out of the list after multiple failed attempts. + + ### Changed + - Improved container logging messages for better readability. + + ### Fixed + - Color setting in configuration now properly works for the embed message + - Fixed the handling of errors and removed from logging to reduce spam. + - Fixed a known logging index error. + +
+
[1.1.0] - 2023-10-16