From 8d6c24a272c022c4f6ff1db550fbdc4be97114c2 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Wed, 9 Dec 2020 16:27:36 +0000 Subject: [PATCH] added serilog and logging --- Directory.Build.props | 10 ++ Dockerfile | 1 + src/Grinder.Common/Grinder.Common.fsproj | 8 +- .../Grinder.ExportTool.fsproj | 5 +- src/Grinder.ExportTool/Program.fs | 8 +- src/Grinder/Commands.fs | 62 ++++++-- src/Grinder/Datastore.fs | 14 +- src/Grinder/Grinder.fsproj | 10 +- src/Grinder/Log.fs | 12 ++ src/Grinder/Program.fs | 135 +++++++++++++++--- tests/Grinder.Tests/Grinder.Tests.fsproj | 5 +- 11 files changed, 209 insertions(+), 61 deletions(-) create mode 100644 Directory.Build.props create mode 100644 src/Grinder/Log.fs diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..e3d6f38 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,10 @@ + + + net5 + false + true + true + true + preview + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 00fb3de..2277ceb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ COPY src/Grinder.Common/Grinder.Common.fsproj ./src/Grinder.Common/Grinder.Commo COPY src/Grinder.DataAccess/Grinder.DataAccess.csproj ./src/Grinder.DataAccess/Grinder.DataAccess.csproj COPY tests/Grinder.Tests/Grinder.Tests.fsproj ./tests/Grinder.Tests/Grinder.Tests.fsproj COPY src/Grinder.ExportTool/Grinder.ExportTool.fsproj ./src/Grinder.ExportTool/Grinder.ExportTool.fsproj +COPY Directory.Build.props ./Directory.Build.props RUN dotnet tool restore RUN dotnet restore -r linux-x64 # Then build app diff --git a/src/Grinder.Common/Grinder.Common.fsproj b/src/Grinder.Common/Grinder.Common.fsproj index 0a04844..cf9f1b9 100644 --- a/src/Grinder.Common/Grinder.Common.fsproj +++ b/src/Grinder.Common/Grinder.Common.fsproj @@ -1,17 +1,11 @@  - - net5 - true - true - - - + diff --git a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj index 9532f8d..d91ea6d 100644 --- a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj +++ b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj @@ -2,9 +2,6 @@ Exe - net5 - true - true @@ -13,7 +10,7 @@ - + diff --git a/src/Grinder.ExportTool/Program.fs b/src/Grinder.ExportTool/Program.fs index 51551f2..04c97a0 100644 --- a/src/Grinder.ExportTool/Program.fs +++ b/src/Grinder.ExportTool/Program.fs @@ -64,13 +64,13 @@ let updateAuthorizationState (dialer: Dialer) (authLock: AutoResetEvent) (state: |> Task.Ignore | :? TdApi.AuthorizationState.AuthorizationStateWaitPhoneNumber -> - printfn "Enter phone" + Console.WriteLine "Enter phone" let phone = Console.ReadLine() do! dialer.ExecuteAsync(new TdApi.SetAuthenticationPhoneNumber(PhoneNumber = phone)) |> Task.Ignore | :? TdApi.AuthorizationState.AuthorizationStateWaitCode -> - printfn "Enter code" + Console.WriteLine "Enter code" let code = Console.ReadLine() do! dialer.ExecuteAsync(new TdApi.CheckAuthenticationCode(Code = code)) |> Task.Ignore @@ -143,7 +143,7 @@ let main argv = dialer.ExecuteAsync(new TdApi.GetChats(Limit = 200, OffsetOrder = Int64.MaxValue)) |> Async.AwaitTask - printfn "Starting export of usernames" + Console.WriteLine "Starting export of usernames" let! users = chats.ChatIds |> AsyncSeq.ofSeq @@ -175,6 +175,6 @@ let main argv = |> JsonConvert.SerializeObject |> stream.Write - printfn "Finished export of usernames" + Console.WriteLine "Finished export of usernames" } |> Async.RunSynchronously 0 // return an integer exit code diff --git a/src/Grinder/Commands.fs b/src/Grinder/Commands.fs index 1395ded..c12309d 100644 --- a/src/Grinder/Commands.fs +++ b/src/Grinder/Commands.fs @@ -428,6 +428,12 @@ module Processing = botSettings.AllowedUsers.Set |> Set.contains username |> not + + let logErrors = + Seq.iter (function + | ApiError str + | AdminBanNotAllowedError str -> logErr str + ) match command with | BanCommand context -> @@ -438,8 +444,12 @@ module Processing = if userCanBeBanned user then for chat in botSettings.ChatsToMonitor.Set do yield botApi.BanUserByUsername chat user context.Until.Value - |> Async.Map ^ fun result -> - Result.mapError ApiError result + |> Async.Map ^ Result.mapError + (sprintf "Error on baning user %A in chat %A until %A. %s" + user + chat + context.Until.Value + >> ApiError) else yield sprintf "Cannot ban admin @%s" %user |> createCommandError AdminBanNotAllowedError @@ -449,6 +459,8 @@ module Processing = requests |> Async.Parallel |> Async.Map getErrors + + logErrors errors let message = { Chats = botSettings.ChatsToMonitor.Set @@ -476,9 +488,14 @@ module Processing = let requests = if userCanBeBanned username then [for chat in botSettings.ChatsToMonitor.Set do - yield botApi.BanUserByUserId chat context.UserId (DateTime.UtcNow.AddMonths(13)) - |> Async.Map ^ fun result -> - Result.mapError ApiError result] + let until = DateTime.UtcNow.AddMonths(13) + yield botApi.BanUserByUserId chat context.UserId until + |> Async.Map ^ Result.mapError + (sprintf "Error on baning userId %A in chat %A until %A. %s" + context.UserId + chat + until + >> ApiError)] else [sprintf "Cannot ban admin %s" %username |> createCommandError AdminBanNotAllowedError @@ -488,6 +505,8 @@ module Processing = requests |> Async.Parallel |> Async.Map getErrors + + logErrors errors let message = { Chats = botSettings.ChatsToMonitor.Set @@ -503,16 +522,24 @@ module Processing = [for user in context.Usernames do for chat in botSettings.ChatsToMonitor.Set do yield botApi.UnbanUser chat user - |> Async.Map ^ fun result -> - Result.mapError ApiError result + |> Async.Map ^ Result.mapError + (sprintf "Error on unbaning user %A in chat %A. %s" + user + chat + >> ApiError) yield botApi.UnrestrictUser chat user - |> Async.Map ^ fun result -> - Result.mapError ApiError result] + |> Async.Map ^ Result.mapError + (sprintf "Error on unrestricting user %A in chat %A. %s" + user + chat + >> ApiError)] let! errors = requests |> Async.Parallel |> Async.Map getErrors + logErrors errors + let message = { Chats = botSettings.ChatsToMonitor.Set Usernames = context.Usernames @@ -537,14 +564,19 @@ module Processing = let requests = [for chat in botSettings.ChatsToMonitor.Set do yield botApi.UnbanUser chat username - |> Async.Map ^ fun result -> - Result.mapError ApiError result] + |> Async.Map ^ Result.mapError + (sprintf "Error on unban user %A in chat %A. %s" + username + chat + >> ApiError)] let! errors = requests |> Async.Parallel |> Async.Map getErrors + logErrors errors + let message = { Chats = botSettings.ChatsToMonitor.Set Usernames = Set.empty.Add username @@ -552,9 +584,12 @@ module Processing = return Some <| UnbanOnReplyMessage(context.From, message, errors) | PingCommand context -> + sprintf "Sending PONG to %A" context.ChatId + |> logInfo do! botApi.SendTextMessage context.ChatId "pong" return None | DoNothingCommand -> + logDbg "Do Nothing Command has been successfully processed and we haven't done anything remotely useful" return None } @@ -573,7 +608,12 @@ module Processing = do! Datastore.upsertUsers users do! "Updated user database" |> ApiExt.sendMessage botSettings.ChannelId config + sprintf "Successfully processed admin command for fileId %s" fileId + |> logInfo + | Error e -> + sprintf "Error on processing admin command with fileId: %s. Error: %s" fileId e + |> logErr do! ApiExt.sendMessage botSettings.ChannelId config e } diff --git a/src/Grinder/Datastore.fs b/src/Grinder/Datastore.fs index 9a9e30d..279984c 100644 --- a/src/Grinder/Datastore.fs +++ b/src/Grinder/Datastore.fs @@ -16,10 +16,14 @@ type FindUsernameByUserIdResult = module Datastore = let upsertUsers (users: User seq) = task { - for chunk in Seq.chunkBySize 500 users do - use context = new GrinderContext() - do! context.Users.AddOrUpdateUsers(chunk) - do! context.SaveChangesAsync() |> Task.Ignore + try + for chunk in Seq.chunkBySize 500 users do + use context = new GrinderContext() + do! context.Users.AddOrUpdateUsers(chunk) + do! context.SaveChangesAsync() |> Task.Ignore + with e -> + sprintf "Error on upserting new users %A" users + |> logExn e } |> Async.AwaitTask @@ -52,4 +56,4 @@ open Grinder.Types type IDataAccessApi = abstract member GetUsernameByUserId: TelegramUserId -> Async - abstract member UpsertUsers: User seq -> Async \ No newline at end of file + abstract member UpsertUsers: User seq -> Async \ No newline at end of file diff --git a/src/Grinder/Grinder.fsproj b/src/Grinder/Grinder.fsproj index 2ee0068..f05cd9b 100644 --- a/src/Grinder/Grinder.fsproj +++ b/src/Grinder/Grinder.fsproj @@ -2,16 +2,11 @@ Exe - net5 - false - true - true - true - + @@ -22,9 +17,12 @@ + + + diff --git a/src/Grinder/Log.fs b/src/Grinder/Log.fs new file mode 100644 index 0000000..53fc5ab --- /dev/null +++ b/src/Grinder/Log.fs @@ -0,0 +1,12 @@ +namespace Grinder + +open Serilog + +[] +module Log = + let mutable logger = Unchecked.defaultof + let logInfo (str: string) = logger.Information str + let logExn (exn: exn) (msg: string) = logger.Error(exn, msg) + let logErr (msg: string) = logger.Error msg + let logDbg (str: string) = logger.Debug str + let logFatal (msg: string) (exn: exn) = logger.Fatal(exn, msg) \ No newline at end of file diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index db29908..d1d696f 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -1,7 +1,6 @@ namespace Grinder open System.Net -open System.Threading open Microsoft.Extensions.Configuration open Funogram open Grinder @@ -13,6 +12,7 @@ open Funogram.Bot open Funogram.Types open FunogramExt open System +open Serilog module Program = open System.Net.Http @@ -52,30 +52,69 @@ module Program = let createBotApi config (settings: BotSettings) = { new IBotApi with - member __.DeleteMessage chatId messageId = + member __.DeleteMessage chatId messageId: Async = + sprintf "Deleting message %A in chat %A" messageId chatId + |> logInfo Api.deleteMessage %chatId %messageId |> callApiWithDefaultRetry config - |> Async.Ignore + |> Async.Map (function + | Ok _ -> () // ignoring + | Error apiError -> + sprintf "Error %s with code %d on deleting message %A in chat %A" + apiError.Description + apiError.ErrorCode + messageId + chatId + |> logErr + ) - member __.BanUserByUsername chatUsername userUsername until = + member __.BanUserByUsername chatUsername userUsername until: Async> = + sprintf "Banning user %A by name in chat %A until %A" userUsername chatUsername until + |> logInfo ApiExt.banUserByUsername config %chatUsername %userUsername until - member __.BanUserByUserId chatUsername userId until = + member __.BanUserByUserId chatUsername userId until: Async> = + sprintf "Banning userId %A in chat %A until %A" userId chatUsername until + |> logInfo ApiExt.banUserByUserId config %chatUsername %userId until - member __.UnbanUser chatUsername username = + member __.UnbanUser chatUsername username: Async> = + sprintf "Unbanning user %A in chat %A" username chatUsername + |> logInfo ApiExt.unbanUserByUsername config %chatUsername %username - member __.SendTextMessage chatId text = + member __.SendTextMessage chatId text: Async = + sprintf "Sending {%s} into chat %A" text chatId + |> logInfo ApiExt.sendMessage chatId config text + |> Async.Catch + |> Async.Map (function + | Choice1Of2 () -> () + | Choice2Of2 e -> + sprintf "Error on sending {%s} into chat %A" text chatId + |> logExn e + ) - member __.UnrestrictUser chatUsername username = + member __.UnrestrictUser chatUsername username: Async> = + sprintf "Unrestricting user %A in chat %A"username chatUsername + |> logInfo ApiExt.unrestrictUserByUsername config %chatUsername %username - member __.SendTextToChannel text = + member __.SendTextToChannel text: Async = + sprintf "Sending {%s} into special channel %A" text settings.ChannelId + |> logInfo ApiExt.sendMessage settings.ChannelId config text + |> Async.Catch + |> Async.Map (function + | Choice1Of2 () -> () + | Choice2Of2 e -> + sprintf "Error on sending {%s} into special channel %A" text settings.ChannelId + |> logExn e + ) - member __.PrepareAndDownloadFile fileId = + member __.PrepareAndDownloadFile fileId: Async> = + sprintf "Downloading file %A" fileId + |> logInfo ApiExt.prepareAndDownloadFile config fileId } @@ -86,11 +125,20 @@ module Program = | UsernameFound username -> return Some %(sprintf "@%s" username) | UsernameNotFound -> + sprintf "User %A hasn't been found in Datastore" userId + |> logErr return None } member __.UpsertUsers users = Datastore.upsertUsers users + |> Async.Catch + |> Async.Map (function + | Choice1Of2 () -> () + | Choice2Of2 e -> + sprintf "Error on upserting new users %A" users + |> logExn e + ) } let onUpdate (settings: BotSettings) (botApi: IBotApi) (dataApi: IDataAccessApi) (context: UpdateContext) = @@ -99,10 +147,21 @@ module Program = |> Option.map ^ fun newMessage -> async { match newMessage with | NewAdminPrivateMessage document -> + sprintf "Received: New admin private message with fileId: %s" document.FileId + |> logInfo do! processAdminCommand settings context.Config document.FileId + | NewUsersAdded users -> + sprintf "Received: New users added %A" users + |> logInfo do! processNewUsersCommand users + | NewMessage message -> + sprintf "Received: New message in chat %s from %A " + (defaultArg message.Chat.Title "Unknown") + message.From + |> logInfo + match prepareTextMessage context.Me.Username message with | Some newMessage -> match authorize settings newMessage.FromUsername newMessage.ChatUsername with @@ -111,9 +170,22 @@ module Program = do! command |> Option.map (formatMessage >> botApi.SendTextToChannel) |> Option.defaultValue Async.Unit - | CommandNotAllowed -> () - | None -> () + | CommandNotAllowed -> + sprintf "Command %s NOT allowed for user %A in chat %A" + newMessage.MessageText + newMessage.FromUsername + newMessage.ChatUsername + |> logInfo + | None -> + sprintf "Skipping message %A from %A" message context.Me.Username + |> logDbg + | NewReplyMessage reply -> + sprintf "Received: New reply message in chat %s from %A" + (defaultArg reply.Message.Chat.Title "Unknown") + (reply.Message.From) + |> logInfo + match prepareReplyToMessage context.Me.Username reply with | Some replyMessage -> match authorize settings replyMessage.FromUsername replyMessage.ChatUsername with @@ -122,15 +194,29 @@ module Program = do! command |> Option.map (formatMessage >> botApi.SendTextToChannel) |> Option.defaultValue Async.Unit - | CommandNotAllowed -> () - | None -> () - | IgnoreMessage -> () + | CommandNotAllowed -> + sprintf "Command %s NOT allowed for user %A in chat %A" + replyMessage.MessageText + replyMessage.FromUsername + replyMessage.ChatUsername + |> logInfo + | None -> + sprintf "Skipping message %A from %A" reply context.Me.Username + |> logDbg + | IgnoreMessage -> + sprintf "Ignoring message %A" context.Update.Message + |> logDbg } |> Option.defaultValue Async.Unit } |> Async.Start [] let main _ = + logger <- LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.Console() + .CreateLogger(); + let mutable configBuilder = ConfigurationBuilder() .AddJsonFile("appsettings.json", false, true) @@ -158,8 +244,6 @@ module Program = GrinderContext.MigrateUp() - printfn "Starting bot" - let settings = { Token = config.Token ChatsToMonitor = ChatsToMonitor.Create config.ChatsToMonitor @@ -167,10 +251,20 @@ module Program = ChannelId = %config.ChannelId AdminUserId = %config.AdminUserId } + + string { botConfiguration with Token = "***" } + |> sprintf "Bot Configuration %A" + |> logInfo + + string { settings with Token = "***" } + |> sprintf "Bot Settings %A" + |> logInfo + + logInfo "Starting bot" startBot botConfiguration (onUpdate settings (createBotApi botConfiguration settings) dataApi) None |> Async.Start - printfn "Bot started" + logInfo "Bot started" // Needed for azure web app deploy check. We have to response with anything on port 80 use listener = new HttpListener() @@ -183,7 +277,8 @@ module Program = let ctx = listener.GetContext() let output = ctx.Response.OutputStream output.Write(buffer, 0, buffer.Length) - output.Close(); + output.Close() + logInfo "Sending OK on HTTP request" - printfn "Bot exited" + logInfo "Bot exited" 0 // return an integer exit code \ No newline at end of file diff --git a/tests/Grinder.Tests/Grinder.Tests.fsproj b/tests/Grinder.Tests/Grinder.Tests.fsproj index 24abb3e..87337b2 100644 --- a/tests/Grinder.Tests/Grinder.Tests.fsproj +++ b/tests/Grinder.Tests/Grinder.Tests.fsproj @@ -2,11 +2,8 @@ Exe - net5 false false - true - true @@ -18,7 +15,7 @@ - +