From 883d0dce0a6359862ec1c4002b82b4cf886cf6ff Mon Sep 17 00:00:00 2001 From: Szer Date: Tue, 9 Jun 2020 11:52:02 +0100 Subject: [PATCH 01/15] [WIP] Deploy to Azure (#29) * Moved to netcoreapp3.1, removed arm * Create azure-deploy.yml * Dockerfile is linux-x64 now * Added ping for debug * Added response on port 80 * Commented proxy optional section Co-authored-by: Ayrat Hudaygulov --- .github/workflows/azure-deploy.yml | 30 +++++++++ Dockerfile | 16 ++--- global.json | 2 +- .../Grinder.DataAccess.csproj | 2 +- .../Grinder.ExportTool.fsproj | 2 +- src/Grinder/Commands.fs | 26 ++++++-- src/Grinder/Grinder.fsproj | 5 +- src/Grinder/Program.fs | 64 +++++++++++++------ src/Grinder/appsettings.json | 13 ++-- tests/Grinder.Tests/Grinder.Tests.fsproj | 2 +- 10 files changed, 111 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/azure-deploy.yml diff --git a/.github/workflows/azure-deploy.yml b/.github/workflows/azure-deploy.yml new file mode 100644 index 0000000..05c0a99 --- /dev/null +++ b/.github/workflows/azure-deploy.yml @@ -0,0 +1,30 @@ +name: Deploy docker image + +on: + push: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Docker Login + uses: Azure/docker-login@v1 + with: + username: ${{ secrets.AZURE_DOCKER_REGISTRY_USER }} + password: ${{ secrets.AZURE_DOCKER_REGISTRY_PASSWORD }} + login-server: https://dotnetru.azurecr.io/v1 + + - uses: actions/checkout@v2 + + - name: Build Docker image + run: docker build . -t grinder + + - name: Tag Docker image + run: docker tag grinder dotnetru.azurecr.io/vahter/grinder + + - name: Push Docker image + run: docker push dotnetru.azurecr.io/vahter/grinder + diff --git a/Dockerfile b/Dockerfile index 9bbcce5..e229f7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,4 @@ -FROM resin/rpi-raspbian AS QEMU -# We need this container to copy QEMU files and use at ARM container -# This will fix pipeline crashes while doing smth - -FROM mcr.microsoft.com/dotnet/core/sdk:2.2.203-stretch AS build-dotnet +FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build-dotnet WORKDIR /app # Copy csproj and restore as distinct layers COPY *.sln ./ @@ -17,12 +13,12 @@ COPY src/Grinder/. ./src/Grinder COPY src/Grinder.Common/. ./src/Grinder.Common COPY src/Grinder.DataAccess/. ./src/Grinder.DataAccess WORKDIR /app/src/Grinder -RUN dotnet publish -r linux-arm -c Release -o out +RUN dotnet publish -r linux-x64 -c Release -o out # Build runtime image -FROM mcr.microsoft.com/dotnet/core/runtime:2.2.4-stretch-slim-arm32v7 -# After COPY we can RUN anything without crashes! -COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static +FROM mcr.microsoft.com/dotnet/core/runtime:3.1 + +EXPOSE 80 WORKDIR /app COPY --from=build-dotnet /app/src/Grinder/out . @@ -31,7 +27,5 @@ RUN mkdir -p /etc/grinder && mkdir -p /app/data VOLUME /etc/grinder/ VOLUME /app/data/ -# In final container we don't need this, removing -RUN rm -f /usr/bin/qemu-arm-static ENTRYPOINT ["dotnet", "Grinder.dll"] diff --git a/global.json b/global.json index b1df68f..d0bfd1d 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "2.2.100" + "version": "3.1.300" } } \ No newline at end of file diff --git a/src/Grinder.DataAccess/Grinder.DataAccess.csproj b/src/Grinder.DataAccess/Grinder.DataAccess.csproj index 68b9d7b..23e2b1d 100644 --- a/src/Grinder.DataAccess/Grinder.DataAccess.csproj +++ b/src/Grinder.DataAccess/Grinder.DataAccess.csproj @@ -1,7 +1,7 @@  - netcoreapp2.2 + netcoreapp3.1 true diff --git a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj index 5a79879..6f1332d 100644 --- a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj +++ b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj @@ -2,7 +2,7 @@ Exe - netcoreapp2.2 + netcoreapp3.1 true true diff --git a/src/Grinder/Commands.fs b/src/Grinder/Commands.fs index 8162ea0..fca42c4 100644 --- a/src/Grinder/Commands.fs +++ b/src/Grinder/Commands.fs @@ -56,7 +56,9 @@ module Parser = | Ban of BanDuration | Unban - type Command = Command of Usernames * CommandAction + type Command = + | UserCommand of Usernames * CommandAction + | Ping let str s = pstring s @@ -107,18 +109,22 @@ module Parser = |> List.map attempt |> choice - let pban: Parser = + let pban: Parser = str "ban" .>> spaces >>. (pforeverBan <|> pdistinctTimeFractions) + |>> Ban let punban: Parser = str "unban" .>> spaces >>% Unban + let pping: Parser = + str "ping" >>% Ping + let pcommandAction: Parser = - (pban |>> Ban) <|> punban + pban <|> punban let parseCommand botUsername = pbotUsername botUsername >>. - pipe2 many1Usernames pcommandAction (fun usernames command -> Command(Usernames(usernames), command)) + (pping <|> pipe2 many1Usernames pcommandAction (fun usernames command -> UserCommand(Usernames(usernames), command))) let runCommandParser botUsername str: ParserResult = run (parseCommand botUsername) str @@ -275,12 +281,13 @@ module Processing = ChatId: TelegramChatId Usernames: UserUsername seq } - + type Command = | BanCommand of BanCommandContext | BanOnReplyCommand of ActionOnReplyCommandContext | UnbanCommand of UnbanCommandContext | UnbanOnReplyCommand of ActionOnReplyCommandContext + | PingCommand | DoNothingCommand type CommandError = @@ -374,7 +381,7 @@ module Processing = let parseTextMessage (context: TextMessageContext): Command = match Parser.parse %context.BotUsername context.MessageText with - | BotCommand(Command((Usernames usernames), Ban(duration))) -> + | BotCommand(UserCommand((Usernames usernames), Ban(duration))) -> let usernames = usernames |> Seq.map ^ fun username -> %username @@ -387,7 +394,7 @@ module Processing = Until = duration } BanCommand context - | BotCommand(Command((Usernames usernames), Unban)) -> + | BotCommand(UserCommand((Usernames usernames), Unban)) -> let usernames = usernames |> Seq.map ^ fun username -> %username @@ -399,6 +406,8 @@ module Processing = Usernames = usernames } UnbanCommand context + | BotCommand(Ping) -> + PingCommand | InvalidCommand _ -> DoNothingCommand @@ -538,6 +547,9 @@ module Processing = } return Some <| UnbanOnReplyMessage(context.From, message, errors) + | PingCommand -> + do! botApi.SendTextToChannel "pong" + return None | DoNothingCommand -> return None } diff --git a/src/Grinder/Grinder.fsproj b/src/Grinder/Grinder.fsproj index f507c82..795a112 100644 --- a/src/Grinder/Grinder.fsproj +++ b/src/Grinder/Grinder.fsproj @@ -2,7 +2,7 @@ Exe - netcoreapp2.2 + netcoreapp3.1 false true true @@ -15,6 +15,7 @@ + @@ -40,4 +41,4 @@ - + \ No newline at end of file diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index 0dd8b94..842a06c 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -1,5 +1,7 @@ namespace Grinder +open System.Net +open System.Threading open Microsoft.Extensions.Configuration open Funogram open Grinder @@ -10,6 +12,7 @@ open Funogram.Api open Funogram.Bot open Funogram.Types open FunogramExt +open System module Program = open System.Net.Http @@ -40,10 +43,11 @@ module Program = Bot: BotConfig } - let createHttpClient config = + let createHttpClient (config: Socks5Configuration) = let messageHandler = new HttpClientHandler() - messageHandler.Proxy <- HttpToSocks5Proxy(config.Hostname, config.Port, config.Username, config.Password) - messageHandler.UseProxy <- true + if not (isNull (box config)) then + messageHandler.Proxy <- HttpToSocks5Proxy(config.Hostname, config.Port, config.Username, config.Password) + messageHandler.UseProxy <- true new HttpClient(messageHandler) let createBotApi config (settings: BotSettings) = { @@ -124,11 +128,20 @@ module Program = [] let main _ = - let config = + let mutable configBuilder = ConfigurationBuilder() .AddJsonFile("appsettings.json", false, true) .AddJsonFile("/etc/grinder/appsettings.json", true, true) .AddEnvironmentVariables("Grinder_") + + match Environment.GetEnvironmentVariable("DOTNETRU_APP_CONFIG") with + | null -> () + | connString -> + configBuilder <- configBuilder + .AddAzureAppConfiguration connString + + let config = + configBuilder .Build() .Get() .Bot; @@ -142,23 +155,32 @@ module Program = GrinderContext.MigrateUp() - async { - printfn "Starting bot" - - let settings = { - Token = config.Token - ChatsToMonitor = ChatsToMonitor.Create config.ChatsToMonitor - AllowedUsers = AllowedUsers.Create config.AllowedUsers - ChannelId = %config.ChannelId - AdminUserId = %config.AdminUserId - } - do! startBot botConfiguration (onUpdate settings (createBotApi botConfiguration settings) dataApi) None - |> Async.StartChild - |> Async.Ignore - - printfn "Bot started" - do! Async.Sleep(-1) - } |> Async.RunSynchronously + printfn "Starting bot" + + let settings = { + Token = config.Token + ChatsToMonitor = ChatsToMonitor.Create config.ChatsToMonitor + AllowedUsers = AllowedUsers.Create config.AllowedUsers + ChannelId = %config.ChannelId + AdminUserId = %config.AdminUserId + } + startBot botConfiguration (onUpdate settings (createBotApi botConfiguration settings) dataApi) None + |> Async.Start + + printfn "Bot started" + + // Needed for azure web app deploy check. We have to response with anything on port 80 + use listener = new HttpListener() + listener.Prefixes.Add("http://*:80/") + listener.Start() + + let buffer = System.Text.Encoding.UTF8.GetBytes "OK" + + while true do + let ctx = listener.GetContext() + let output = ctx.Response.OutputStream + output.Write(buffer, 0, buffer.Length) + output.Close(); printfn "Bot exited" 0 // return an integer exit code \ No newline at end of file diff --git a/src/Grinder/appsettings.json b/src/Grinder/appsettings.json index f9c3834..2b88449 100644 --- a/src/Grinder/appsettings.json +++ b/src/Grinder/appsettings.json @@ -3,12 +3,13 @@ "Token": "REPLACEME", "ChannelId": 0, "AdminUserId": 0, - "Socks5Proxy": { - "Hostname": "REPLACEME", - "Port": 0, - "Username": "REPLACEME", - "Password": "REPLACEME" - }, +// uncomment if you need proxy +// "Socks5Proxy": { +// "Hostname": "REPLACEME", +// "Port": 0, +// "Username": "REPLACEME", +// "Password": "REPLACEME" +// }, "ChatsToMonitor": [ "REPLACEME" ], diff --git a/tests/Grinder.Tests/Grinder.Tests.fsproj b/tests/Grinder.Tests/Grinder.Tests.fsproj index 4d659f3..7a69813 100644 --- a/tests/Grinder.Tests/Grinder.Tests.fsproj +++ b/tests/Grinder.Tests/Grinder.Tests.fsproj @@ -2,7 +2,7 @@ Exe - netcoreapp2.2 + netcoreapp3.1 false false true From 4de188763d2a0bfad3a17f5f5c5902f0eae26578 Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Tue, 9 Jun 2020 22:38:59 +0300 Subject: [PATCH 02/15] Pong to chat, add parser test for ping, fix tests (#30) * fix tests, add test for ping * reply with pong to chat --- src/Grinder/Commands.fs | 12 ++++++++---- src/Grinder/Program.fs | 3 +++ src/Grinder/Types.fs | 1 + tests/Grinder.Tests/ParserTests.fs | 17 ++++++++++++++--- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/Grinder/Commands.fs b/src/Grinder/Commands.fs index fca42c4..1395ded 100644 --- a/src/Grinder/Commands.fs +++ b/src/Grinder/Commands.fs @@ -282,12 +282,16 @@ module Processing = Usernames: UserUsername seq } + type PingContext = { + ChatId: TelegramChatId + } + type Command = | BanCommand of BanCommandContext | BanOnReplyCommand of ActionOnReplyCommandContext | UnbanCommand of UnbanCommandContext | UnbanOnReplyCommand of ActionOnReplyCommandContext - | PingCommand + | PingCommand of PingContext | DoNothingCommand type CommandError = @@ -407,7 +411,7 @@ module Processing = } UnbanCommand context | BotCommand(Ping) -> - PingCommand + PingCommand { ChatId = %context.Message.Chat.Id } | InvalidCommand _ -> DoNothingCommand @@ -547,8 +551,8 @@ module Processing = } return Some <| UnbanOnReplyMessage(context.From, message, errors) - | PingCommand -> - do! botApi.SendTextToChannel "pong" + | PingCommand context -> + do! botApi.SendTextMessage context.ChatId "pong" return None | DoNothingCommand -> return None diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index 842a06c..db29908 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -66,6 +66,9 @@ module Program = member __.UnbanUser chatUsername username = ApiExt.unbanUserByUsername config %chatUsername %username + member __.SendTextMessage chatId text = + ApiExt.sendMessage chatId config text + member __.UnrestrictUser chatUsername username = ApiExt.unrestrictUserByUsername config %chatUsername %username diff --git a/src/Grinder/Types.fs b/src/Grinder/Types.fs index e7f8688..afef728 100644 --- a/src/Grinder/Types.fs +++ b/src/Grinder/Types.fs @@ -97,4 +97,5 @@ type IBotApi = abstract member UnbanUser: ChatUsername -> UserUsername -> Async> abstract member UnrestrictUser: ChatUsername -> UserUsername -> Async> abstract member SendTextToChannel: string -> Async + abstract member SendTextMessage: TelegramChatId -> string -> Async abstract member PrepareAndDownloadFile: string -> Async> \ No newline at end of file diff --git a/tests/Grinder.Tests/ParserTests.fs b/tests/Grinder.Tests/ParserTests.fs index 8cbdeb4..4e2a7ac 100644 --- a/tests/Grinder.Tests/ParserTests.fs +++ b/tests/Grinder.Tests/ParserTests.fs @@ -109,7 +109,7 @@ let ``pforeverBan parses forever ban as forever text`` () = [] let ``parseCommand returns correct command for ban`` (command) = match run (parseCommand "@bot") command with - | Success(Command(Usernames(usernames), Ban(Timed(duration))), _, _) -> + | Success(UserCommand(Usernames(usernames), Ban(Timed(duration))), _, _) -> let time = DateTime.UtcNow.AddMonths(12).AddDays(1.) Assert.Equal(time.ToString("yyyyMMddTHH:mm"), duration.ToString("yyyyMMddTHH:mm")) Assert.Equal(["@first"; "@second"], usernames) @@ -125,18 +125,29 @@ let ``parseCommand returns correct command for ban`` (command) = [] let ``parseCommand returns correct command for forever ban`` (command: string) = match run (parseCommand "@bot") command with - | Success(Command(Usernames(usernames), Ban(Forever)), _, _) -> + | Success(UserCommand(Usernames(usernames), Ban(Forever)), _, _) -> Assert.Equal(["@first"; "@second"], usernames) | Failure(error, _, _ ) -> Assert.FailWithMessage(error) | _ -> Assert.Fail() + +[] +let ``parseCommand returns correct command for ping`` () = + let command = "@bot ping" + match run (parseCommand "@bot") command with + | Success(Ping, _, _) -> + Assert.Success() + | Failure(error, _, _ ) -> + Assert.FailWithMessage(error) + | _ -> + Assert.Fail() [] let ``parseCommand returns correct command for unban`` () = let command = "@bot @first @second unban" match run (parseCommand "@bot") command with - | Success(Command(Usernames(usernames), Unban), _, _) -> + | Success(UserCommand(Usernames(usernames), Unban), _, _) -> Assert.Equal(["@first"; "@second"], usernames) | Failure(error, _, _ ) -> Assert.FailWithMessage(error) From b336c53e53cd8e8ea912f8f023da7123639540fb Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Thu, 11 Jun 2020 00:58:07 +0300 Subject: [PATCH 03/15] Add github test action --- .github/workflows/dotnet-test.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/dotnet-test.yml diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml new file mode 100644 index 0000000..c617fb8 --- /dev/null +++ b/.github/workflows/dotnet-test.yml @@ -0,0 +1,23 @@ +name: .NET Core + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup .NET Core + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 3.1.101 + - name: Restore tools + run: dotnet tool restore + - name: Test + run: dotnet test --verbosity normal From 6953b0a2cb4e7e661cdcfe31bd5a197a08e5cbd9 Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Thu, 11 Jun 2020 00:59:10 +0300 Subject: [PATCH 04/15] Fix test sdk --- .github/workflows/dotnet-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml index c617fb8..5ef297a 100644 --- a/.github/workflows/dotnet-test.yml +++ b/.github/workflows/dotnet-test.yml @@ -16,7 +16,7 @@ jobs: - name: Setup .NET Core uses: actions/setup-dotnet@v1 with: - dotnet-version: 3.1.101 + dotnet-version: 3.1.300 - name: Restore tools run: dotnet tool restore - name: Test From c83c9ab431b5b3bc252a18deefb3224a67b407a3 Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Thu, 11 Jun 2020 01:04:38 +0300 Subject: [PATCH 05/15] Delete azure-pipelines.yml --- azure-pipelines.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 4efb33e..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,35 +0,0 @@ -trigger: - - master - -pool: - vmImage: 'ubuntu-16.04' - -variables: - imageName: $(image-name) - -steps: - -- script: docker run --rm --privileged multiarch/qemu-user-static:register --reset - displayName: Run QEMU - -- task: Docker@2 - displayName: Build + Push (arm) - inputs: - containerRegistry: $(docker-hub) - repository: $(docker-user)/$(imageName) - command: buildAndPush - tags: | - latest-arm - arm - -- task: Docker@2 - displayName: Build + Push (x86) - inputs: - containerRegistry: $(docker-hub) - repository: $(docker-user)/$(imageName) - command: buildAndPush - Dockerfile: '**/x86.Dockerfile' - tags: | - latest - latest-x86 - x86 \ No newline at end of file From 4bca43bc4b20aea126b7d06b23480e8fcc3d3faf Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Thu, 11 Jun 2020 01:08:21 +0300 Subject: [PATCH 06/15] Ignore .md files in deploy workflow --- .github/workflows/azure-deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/azure-deploy.yml b/.github/workflows/azure-deploy.yml index 05c0a99..32c6e18 100644 --- a/.github/workflows/azure-deploy.yml +++ b/.github/workflows/azure-deploy.yml @@ -3,6 +3,8 @@ name: Deploy docker image on: push: branches: [ master ] + paths-ignore: + - "**.md" jobs: build: From 9c469327250a5f110b1cd9b5bd60110ba72d2e1c Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Thu, 11 Jun 2020 01:23:54 +0300 Subject: [PATCH 07/15] Delete old dockerfile --- x86.Dockerfile | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 x86.Dockerfile diff --git a/x86.Dockerfile b/x86.Dockerfile deleted file mode 100644 index a20a9b2..0000000 --- a/x86.Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:2.2-alpine AS build-dotnet -WORKDIR /app -# Copy csproj and restore as distinct layers -COPY *.sln ./ -COPY src/Grinder/Grinder.fsproj ./src/Grinder/Grinder.fsproj -COPY src/Grinder.Common/Grinder.Common.fsproj ./src/Grinder.Common/Grinder.Common.fsproj -COPY src/Grinder.DataAccess/Grinder.DataAccess.csproj ./src/Grinder.DataAccess/Grinder.DataAccess.csproj -COPY tests/Grinder.Tests/Grinder.Tests.fsproj ./tests/Grinder.Tests/Grinder.Tests.fsproj -COPY src/Grinder.ExportTool/Grinder.ExportTool.fsproj ./src/Grinder.ExportTool/Grinder.ExportTool.fsproj -RUN dotnet restore -# Then build app -COPY src/Grinder/. ./src/Grinder -COPY src/Grinder.Common/. ./src/Grinder.Common -COPY src/Grinder.DataAccess/. ./src/Grinder.DataAccess -WORKDIR /app/src/Grinder -RUN dotnet publish -c Release -o out - -# Build runtime image -FROM mcr.microsoft.com/dotnet/core/runtime:2.2-alpine -WORKDIR /app -COPY --from=build-dotnet /app/src/Grinder/out . - -RUN mkdir -p /etc/grinder && mkdir -p /app/data - -VOLUME /etc/grinder/ -VOLUME /app/data/ - -ENTRYPOINT ["dotnet", "Grinder.dll"] From 07b408c4a4c3b64b4fbe7924524c8bb6725f604c Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Thu, 11 Jun 2020 01:27:39 +0300 Subject: [PATCH 08/15] Build before test in action --- .github/workflows/dotnet-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml index 5ef297a..0b124fc 100644 --- a/.github/workflows/dotnet-test.yml +++ b/.github/workflows/dotnet-test.yml @@ -19,5 +19,7 @@ jobs: dotnet-version: 3.1.300 - name: Restore tools run: dotnet tool restore + - name: Build + run: dotnet build -c Release - name: Test - run: dotnet test --verbosity normal + run: dotnet test -c Release --no-restore --verbosity normal From 8d29618d20311a123721da6056b042799a239f46 Mon Sep 17 00:00:00 2001 From: YogurtTheHorse Date: Thu, 22 Oct 2020 00:59:22 +0300 Subject: [PATCH 09/15] Fix `Couldn't resolve username @name` and add option to disable web server run (#34) * Update .net sdk * Add listen endpoint to settings * Disable azure webserver * Update docs * Fix username searching * Revert sdk version * Replace invariant string comparison with StringComparison.InvariantCultureIgnoreCase * Replace RunAzureWebServer with DisableAzureWebServer With such change no config changes will be required on server --- README.md | 9 +++++---- src/Grinder/Datastore.fs | 3 ++- src/Grinder/Program.fs | 28 ++++++++++++++++------------ src/Grinder/appsettings.json | 1 + 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 68d6130..c3440cc 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ You can choose two ways to configure grinder but keep in mind about configuratio "Token": "test", "ChannelId": 111, "AdminUserId": 123, + "RunAzureWebServer": true, "Socks5Proxy": { "Hostname": "Hostme", "Port": 1337, @@ -71,8 +72,8 @@ You can choose two ways to configure grinder but keep in mind about configuratio "Password": "Secrete" }, "ChatsToMonitor": [ - "Sample", - "Text" + "@Sample", + "@Text" ], "AllowedUsers": [ "Pasha", @@ -102,8 +103,8 @@ Grinder_Bot__AllowedUsers__0=Pasha Grinder_Bot__AllowedUsers__1=Technique #ChatsToMonitor - chats where bot will read messages and replies -Grinder_Bot__ChatsToMonitor__0=Sample -Grinder_Bot__ChatsToMonitor__1=Text +Grinder_Bot__ChatsToMonitor__0=@Sample +Grinder_Bot__ChatsToMonitor__1=@Text #AdminUserId - telegram user id who will be able to send private messages to bot Grinder_Bot__AdminUserId=123 diff --git a/src/Grinder/Datastore.fs b/src/Grinder/Datastore.fs index 9a9e30d..f1046c2 100644 --- a/src/Grinder/Datastore.fs +++ b/src/Grinder/Datastore.fs @@ -1,5 +1,6 @@ namespace Grinder +open System open FSharp.Control.Tasks.V2 open Microsoft.EntityFrameworkCore open Grinder.DataAccess @@ -28,7 +29,7 @@ module Datastore = use context = new GrinderContext() let! user = context.Users - .FirstOrDefaultAsync(fun u -> u.Username = username.TrimStart('@')) + .FirstOrDefaultAsync(fun u -> u.Username.Equals(username.TrimStart('@'), StringComparison.InvariantCultureIgnoreCase)) return user |> Option.ofObj |> Option.fold (fun _ u -> UserIdFound u.UserId) UserIdNotFound diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index db29908..73e9106 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -36,6 +36,7 @@ module Program = AllowedUsers: string array ChannelId: int64 AdminUserId: int64 + DisableAzureWebServer: bool } [] @@ -172,18 +173,21 @@ module Program = printfn "Bot started" - // Needed for azure web app deploy check. We have to response with anything on port 80 - use listener = new HttpListener() - listener.Prefixes.Add("http://*:80/") - listener.Start() - - let buffer = System.Text.Encoding.UTF8.GetBytes "OK" - - while true do - let ctx = listener.GetContext() - let output = ctx.Response.OutputStream - output.Write(buffer, 0, buffer.Length) - output.Close(); + if not config.DisableAzureWebServer then + // Needed for azure web app deploy check. We have to response with anything on port 80 + use listener = new HttpListener() + listener.Prefixes.Add("http://*:80/") + listener.Start() + + let buffer = System.Text.Encoding.UTF8.GetBytes "OK" + + while true do + let ctx = listener.GetContext() + let output = ctx.Response.OutputStream + output.Write(buffer, 0, buffer.Length) + output.Close() + else + Console.ReadLine () |> ignore printfn "Bot exited" 0 // return an integer exit code \ No newline at end of file diff --git a/src/Grinder/appsettings.json b/src/Grinder/appsettings.json index 2b88449..1e1c731 100644 --- a/src/Grinder/appsettings.json +++ b/src/Grinder/appsettings.json @@ -3,6 +3,7 @@ "Token": "REPLACEME", "ChannelId": 0, "AdminUserId": 0, + "DisableAzureWebServer": true, // uncomment if you need proxy // "Socks5Proxy": { // "Hostname": "REPLACEME", From e9c9a98cf7aa4846f53d86a7ef9e7ca17b8ad514 Mon Sep 17 00:00:00 2001 From: Vlad Khapin Date: Fri, 23 Oct 2020 19:16:07 +0300 Subject: [PATCH 10/15] Revert "Fix `Couldn't resolve username @name` and add option to disable web server run (#34)" (#35) This reverts commit 8d29618d20311a123721da6056b042799a239f46. --- README.md | 9 ++++----- src/Grinder/Datastore.fs | 3 +-- src/Grinder/Program.fs | 28 ++++++++++++---------------- src/Grinder/appsettings.json | 1 - 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c3440cc..68d6130 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,6 @@ You can choose two ways to configure grinder but keep in mind about configuratio "Token": "test", "ChannelId": 111, "AdminUserId": 123, - "RunAzureWebServer": true, "Socks5Proxy": { "Hostname": "Hostme", "Port": 1337, @@ -72,8 +71,8 @@ You can choose two ways to configure grinder but keep in mind about configuratio "Password": "Secrete" }, "ChatsToMonitor": [ - "@Sample", - "@Text" + "Sample", + "Text" ], "AllowedUsers": [ "Pasha", @@ -103,8 +102,8 @@ Grinder_Bot__AllowedUsers__0=Pasha Grinder_Bot__AllowedUsers__1=Technique #ChatsToMonitor - chats where bot will read messages and replies -Grinder_Bot__ChatsToMonitor__0=@Sample -Grinder_Bot__ChatsToMonitor__1=@Text +Grinder_Bot__ChatsToMonitor__0=Sample +Grinder_Bot__ChatsToMonitor__1=Text #AdminUserId - telegram user id who will be able to send private messages to bot Grinder_Bot__AdminUserId=123 diff --git a/src/Grinder/Datastore.fs b/src/Grinder/Datastore.fs index f1046c2..9a9e30d 100644 --- a/src/Grinder/Datastore.fs +++ b/src/Grinder/Datastore.fs @@ -1,6 +1,5 @@ namespace Grinder -open System open FSharp.Control.Tasks.V2 open Microsoft.EntityFrameworkCore open Grinder.DataAccess @@ -29,7 +28,7 @@ module Datastore = use context = new GrinderContext() let! user = context.Users - .FirstOrDefaultAsync(fun u -> u.Username.Equals(username.TrimStart('@'), StringComparison.InvariantCultureIgnoreCase)) + .FirstOrDefaultAsync(fun u -> u.Username = username.TrimStart('@')) return user |> Option.ofObj |> Option.fold (fun _ u -> UserIdFound u.UserId) UserIdNotFound diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index 73e9106..db29908 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -36,7 +36,6 @@ module Program = AllowedUsers: string array ChannelId: int64 AdminUserId: int64 - DisableAzureWebServer: bool } [] @@ -173,21 +172,18 @@ module Program = printfn "Bot started" - if not config.DisableAzureWebServer then - // Needed for azure web app deploy check. We have to response with anything on port 80 - use listener = new HttpListener() - listener.Prefixes.Add("http://*:80/") - listener.Start() - - let buffer = System.Text.Encoding.UTF8.GetBytes "OK" - - while true do - let ctx = listener.GetContext() - let output = ctx.Response.OutputStream - output.Write(buffer, 0, buffer.Length) - output.Close() - else - Console.ReadLine () |> ignore + // Needed for azure web app deploy check. We have to response with anything on port 80 + use listener = new HttpListener() + listener.Prefixes.Add("http://*:80/") + listener.Start() + + let buffer = System.Text.Encoding.UTF8.GetBytes "OK" + + while true do + let ctx = listener.GetContext() + let output = ctx.Response.OutputStream + output.Write(buffer, 0, buffer.Length) + output.Close(); printfn "Bot exited" 0 // return an integer exit code \ No newline at end of file diff --git a/src/Grinder/appsettings.json b/src/Grinder/appsettings.json index 1e1c731..2b88449 100644 --- a/src/Grinder/appsettings.json +++ b/src/Grinder/appsettings.json @@ -3,7 +3,6 @@ "Token": "REPLACEME", "ChannelId": 0, "AdminUserId": 0, - "DisableAzureWebServer": true, // uncomment if you need proxy // "Socks5Proxy": { // "Hostname": "REPLACEME", From 203763e6861d436c714508bbcdd1bf64bf691a11 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Thu, 10 Dec 2020 13:18:37 +0000 Subject: [PATCH 11/15] tests now also comes through docker to test it (#38) Co-authored-by: Ayrat Hudaygulov --- .github/workflows/dotnet-test.yml | 20 +++++++------------- .gitignore | 1 + Dockerfile | 10 +++++++--- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml index 0b124fc..47e4111 100644 --- a/.github/workflows/dotnet-test.yml +++ b/.github/workflows/dotnet-test.yml @@ -1,25 +1,19 @@ -name: .NET Core +name: Build & Test on: push: branches: [ master ] + paths-ignore: + - "**.md" pull_request: branches: [ master ] + paths-ignore: + - "**.md" jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v2 - - name: Setup .NET Core - uses: actions/setup-dotnet@v1 - with: - dotnet-version: 3.1.300 - - name: Restore tools - run: dotnet tool restore - - name: Build - run: dotnet build -c Release - - name: Test - run: dotnet test -c Release --no-restore --verbosity normal + - name: Build & Test + run: docker build . diff --git a/.gitignore b/.gitignore index dfd2a0c..0b9cf70 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ +out/ # Visual Studio 2015 cache/options directory .vs/ diff --git a/Dockerfile b/Dockerfile index e229f7c..e15e8cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build-dotnet +FROM mcr.microsoft.com/dotnet/sdk:3.1 AS build-dotnet WORKDIR /app # Copy csproj and restore as distinct layers COPY *.sln ./ @@ -7,16 +7,20 @@ COPY src/Grinder.Common/Grinder.Common.fsproj ./src/Grinder.Common/Grinder.Commo COPY src/Grinder.DataAccess/Grinder.DataAccess.csproj ./src/Grinder.DataAccess/Grinder.DataAccess.csproj COPY tests/Grinder.Tests/Grinder.Tests.fsproj ./tests/Grinder.Tests/Grinder.Tests.fsproj COPY src/Grinder.ExportTool/Grinder.ExportTool.fsproj ./src/Grinder.ExportTool/Grinder.ExportTool.fsproj -RUN dotnet restore +RUN dotnet tool restore +RUN dotnet restore -r linux-x64 # Then build app COPY src/Grinder/. ./src/Grinder COPY src/Grinder.Common/. ./src/Grinder.Common COPY src/Grinder.DataAccess/. ./src/Grinder.DataAccess WORKDIR /app/src/Grinder +# Build&Test +RUN dotnet test -c Release -r linux-x64 -o out --no-restore --verbosity normal +# Publish RUN dotnet publish -r linux-x64 -c Release -o out # Build runtime image -FROM mcr.microsoft.com/dotnet/core/runtime:3.1 +FROM mcr.microsoft.com/dotnet/runtime:3.1 EXPOSE 80 From 0ddfd3d32da4f14eeeb43a4dde6a659b86577882 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Thu, 10 Dec 2020 13:24:02 +0000 Subject: [PATCH 12/15] Merge pull request #36 * migrating to net5 --- Dockerfile | 4 ++-- Grinder.sln | 2 -- README.md | 6 ++++-- global.json | 2 +- src/Grinder.Common/Grinder.Common.fsproj | 4 ++-- src/Grinder.DataAccess/Grinder.DataAccess.csproj | 2 +- src/Grinder.ExportTool/Grinder.ExportTool.fsproj | 4 ++-- src/Grinder/Grinder.fsproj | 4 ++-- tests/Grinder.Tests/Grinder.Tests.fsproj | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index e15e8cc..00fb3de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:3.1 AS build-dotnet +FROM mcr.microsoft.com/dotnet/sdk:5.0 AS build-dotnet WORKDIR /app # Copy csproj and restore as distinct layers COPY *.sln ./ @@ -20,7 +20,7 @@ RUN dotnet test -c Release -r linux-x64 -o out --no-restore --verbosity normal RUN dotnet publish -r linux-x64 -c Release -o out # Build runtime image -FROM mcr.microsoft.com/dotnet/runtime:3.1 +FROM mcr.microsoft.com/dotnet/runtime:5.0 EXPOSE 80 diff --git a/Grinder.sln b/Grinder.sln index 9fd18a5..c2109b3 100644 --- a/Grinder.sln +++ b/Grinder.sln @@ -17,8 +17,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{25990364-33CE- ProjectSection(SolutionItems) = preProject docker-compose.yml = docker-compose.yml Dockerfile = Dockerfile - azure-pipelines.yml = azure-pipelines.yml - x86.Dockerfile = x86.Dockerfile EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{3AB38C70-1DF4-48D9-992B-B7BFBFDDFD7A}" diff --git a/README.md b/README.md index 68d6130..f160064 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,11 @@ Telegram bot for chats administration. ## Configuration and build -Just open project with Rider/VS 2019 and run build +- Install Net5 SDK -or use dotnet cli +- Open project with Rider/VS 2019 and run build + +- or use dotnet cli ```bash dotnet build diff --git a/global.json b/global.json index d0bfd1d..2ed6663 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "3.1.300" + "version": "5.0.101" } } \ No newline at end of file diff --git a/src/Grinder.Common/Grinder.Common.fsproj b/src/Grinder.Common/Grinder.Common.fsproj index e9ae9d0..0a04844 100644 --- a/src/Grinder.Common/Grinder.Common.fsproj +++ b/src/Grinder.Common/Grinder.Common.fsproj @@ -1,7 +1,7 @@  - netstandard2.0 + net5 true true @@ -11,7 +11,7 @@ - + diff --git a/src/Grinder.DataAccess/Grinder.DataAccess.csproj b/src/Grinder.DataAccess/Grinder.DataAccess.csproj index 23e2b1d..c612362 100644 --- a/src/Grinder.DataAccess/Grinder.DataAccess.csproj +++ b/src/Grinder.DataAccess/Grinder.DataAccess.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net5 true diff --git a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj index 6f1332d..9532f8d 100644 --- a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj +++ b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net5 true true @@ -13,7 +13,7 @@ - + diff --git a/src/Grinder/Grinder.fsproj b/src/Grinder/Grinder.fsproj index 795a112..2ee0068 100644 --- a/src/Grinder/Grinder.fsproj +++ b/src/Grinder/Grinder.fsproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net5 false true true @@ -11,7 +11,7 @@ - + diff --git a/tests/Grinder.Tests/Grinder.Tests.fsproj b/tests/Grinder.Tests/Grinder.Tests.fsproj index 7a69813..24abb3e 100644 --- a/tests/Grinder.Tests/Grinder.Tests.fsproj +++ b/tests/Grinder.Tests/Grinder.Tests.fsproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net5 false false true @@ -18,7 +18,7 @@ - + From 21fb856c8c1fc5afab6d08eeb375798fffe22deb Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Thu, 10 Dec 2020 13:27:42 +0000 Subject: [PATCH 13/15] added serilog and logging (#37) Co-authored-by: Ayrat Hudaygulov --- Directory.Build.props | 10 ++ Dockerfile | 1 + src/Grinder.Common/Grinder.Common.fsproj | 8 +- .../Grinder.ExportTool.fsproj | 5 +- src/Grinder.ExportTool/Program.fs | 8 +- src/Grinder/Commands.fs | 62 ++++++-- src/Grinder/Datastore.fs | 14 +- src/Grinder/Grinder.fsproj | 10 +- src/Grinder/Log.fs | 12 ++ src/Grinder/Program.fs | 135 +++++++++++++++--- tests/Grinder.Tests/Grinder.Tests.fsproj | 5 +- 11 files changed, 209 insertions(+), 61 deletions(-) create mode 100644 Directory.Build.props create mode 100644 src/Grinder/Log.fs diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..e3d6f38 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,10 @@ + + + net5 + false + true + true + true + preview + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 00fb3de..2277ceb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ COPY src/Grinder.Common/Grinder.Common.fsproj ./src/Grinder.Common/Grinder.Commo COPY src/Grinder.DataAccess/Grinder.DataAccess.csproj ./src/Grinder.DataAccess/Grinder.DataAccess.csproj COPY tests/Grinder.Tests/Grinder.Tests.fsproj ./tests/Grinder.Tests/Grinder.Tests.fsproj COPY src/Grinder.ExportTool/Grinder.ExportTool.fsproj ./src/Grinder.ExportTool/Grinder.ExportTool.fsproj +COPY Directory.Build.props ./Directory.Build.props RUN dotnet tool restore RUN dotnet restore -r linux-x64 # Then build app diff --git a/src/Grinder.Common/Grinder.Common.fsproj b/src/Grinder.Common/Grinder.Common.fsproj index 0a04844..cf9f1b9 100644 --- a/src/Grinder.Common/Grinder.Common.fsproj +++ b/src/Grinder.Common/Grinder.Common.fsproj @@ -1,17 +1,11 @@  - - net5 - true - true - - - + diff --git a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj index 9532f8d..d91ea6d 100644 --- a/src/Grinder.ExportTool/Grinder.ExportTool.fsproj +++ b/src/Grinder.ExportTool/Grinder.ExportTool.fsproj @@ -2,9 +2,6 @@ Exe - net5 - true - true @@ -13,7 +10,7 @@ - + diff --git a/src/Grinder.ExportTool/Program.fs b/src/Grinder.ExportTool/Program.fs index 51551f2..04c97a0 100644 --- a/src/Grinder.ExportTool/Program.fs +++ b/src/Grinder.ExportTool/Program.fs @@ -64,13 +64,13 @@ let updateAuthorizationState (dialer: Dialer) (authLock: AutoResetEvent) (state: |> Task.Ignore | :? TdApi.AuthorizationState.AuthorizationStateWaitPhoneNumber -> - printfn "Enter phone" + Console.WriteLine "Enter phone" let phone = Console.ReadLine() do! dialer.ExecuteAsync(new TdApi.SetAuthenticationPhoneNumber(PhoneNumber = phone)) |> Task.Ignore | :? TdApi.AuthorizationState.AuthorizationStateWaitCode -> - printfn "Enter code" + Console.WriteLine "Enter code" let code = Console.ReadLine() do! dialer.ExecuteAsync(new TdApi.CheckAuthenticationCode(Code = code)) |> Task.Ignore @@ -143,7 +143,7 @@ let main argv = dialer.ExecuteAsync(new TdApi.GetChats(Limit = 200, OffsetOrder = Int64.MaxValue)) |> Async.AwaitTask - printfn "Starting export of usernames" + Console.WriteLine "Starting export of usernames" let! users = chats.ChatIds |> AsyncSeq.ofSeq @@ -175,6 +175,6 @@ let main argv = |> JsonConvert.SerializeObject |> stream.Write - printfn "Finished export of usernames" + Console.WriteLine "Finished export of usernames" } |> Async.RunSynchronously 0 // return an integer exit code diff --git a/src/Grinder/Commands.fs b/src/Grinder/Commands.fs index 1395ded..c12309d 100644 --- a/src/Grinder/Commands.fs +++ b/src/Grinder/Commands.fs @@ -428,6 +428,12 @@ module Processing = botSettings.AllowedUsers.Set |> Set.contains username |> not + + let logErrors = + Seq.iter (function + | ApiError str + | AdminBanNotAllowedError str -> logErr str + ) match command with | BanCommand context -> @@ -438,8 +444,12 @@ module Processing = if userCanBeBanned user then for chat in botSettings.ChatsToMonitor.Set do yield botApi.BanUserByUsername chat user context.Until.Value - |> Async.Map ^ fun result -> - Result.mapError ApiError result + |> Async.Map ^ Result.mapError + (sprintf "Error on baning user %A in chat %A until %A. %s" + user + chat + context.Until.Value + >> ApiError) else yield sprintf "Cannot ban admin @%s" %user |> createCommandError AdminBanNotAllowedError @@ -449,6 +459,8 @@ module Processing = requests |> Async.Parallel |> Async.Map getErrors + + logErrors errors let message = { Chats = botSettings.ChatsToMonitor.Set @@ -476,9 +488,14 @@ module Processing = let requests = if userCanBeBanned username then [for chat in botSettings.ChatsToMonitor.Set do - yield botApi.BanUserByUserId chat context.UserId (DateTime.UtcNow.AddMonths(13)) - |> Async.Map ^ fun result -> - Result.mapError ApiError result] + let until = DateTime.UtcNow.AddMonths(13) + yield botApi.BanUserByUserId chat context.UserId until + |> Async.Map ^ Result.mapError + (sprintf "Error on baning userId %A in chat %A until %A. %s" + context.UserId + chat + until + >> ApiError)] else [sprintf "Cannot ban admin %s" %username |> createCommandError AdminBanNotAllowedError @@ -488,6 +505,8 @@ module Processing = requests |> Async.Parallel |> Async.Map getErrors + + logErrors errors let message = { Chats = botSettings.ChatsToMonitor.Set @@ -503,16 +522,24 @@ module Processing = [for user in context.Usernames do for chat in botSettings.ChatsToMonitor.Set do yield botApi.UnbanUser chat user - |> Async.Map ^ fun result -> - Result.mapError ApiError result + |> Async.Map ^ Result.mapError + (sprintf "Error on unbaning user %A in chat %A. %s" + user + chat + >> ApiError) yield botApi.UnrestrictUser chat user - |> Async.Map ^ fun result -> - Result.mapError ApiError result] + |> Async.Map ^ Result.mapError + (sprintf "Error on unrestricting user %A in chat %A. %s" + user + chat + >> ApiError)] let! errors = requests |> Async.Parallel |> Async.Map getErrors + logErrors errors + let message = { Chats = botSettings.ChatsToMonitor.Set Usernames = context.Usernames @@ -537,14 +564,19 @@ module Processing = let requests = [for chat in botSettings.ChatsToMonitor.Set do yield botApi.UnbanUser chat username - |> Async.Map ^ fun result -> - Result.mapError ApiError result] + |> Async.Map ^ Result.mapError + (sprintf "Error on unban user %A in chat %A. %s" + username + chat + >> ApiError)] let! errors = requests |> Async.Parallel |> Async.Map getErrors + logErrors errors + let message = { Chats = botSettings.ChatsToMonitor.Set Usernames = Set.empty.Add username @@ -552,9 +584,12 @@ module Processing = return Some <| UnbanOnReplyMessage(context.From, message, errors) | PingCommand context -> + sprintf "Sending PONG to %A" context.ChatId + |> logInfo do! botApi.SendTextMessage context.ChatId "pong" return None | DoNothingCommand -> + logDbg "Do Nothing Command has been successfully processed and we haven't done anything remotely useful" return None } @@ -573,7 +608,12 @@ module Processing = do! Datastore.upsertUsers users do! "Updated user database" |> ApiExt.sendMessage botSettings.ChannelId config + sprintf "Successfully processed admin command for fileId %s" fileId + |> logInfo + | Error e -> + sprintf "Error on processing admin command with fileId: %s. Error: %s" fileId e + |> logErr do! ApiExt.sendMessage botSettings.ChannelId config e } diff --git a/src/Grinder/Datastore.fs b/src/Grinder/Datastore.fs index 9a9e30d..279984c 100644 --- a/src/Grinder/Datastore.fs +++ b/src/Grinder/Datastore.fs @@ -16,10 +16,14 @@ type FindUsernameByUserIdResult = module Datastore = let upsertUsers (users: User seq) = task { - for chunk in Seq.chunkBySize 500 users do - use context = new GrinderContext() - do! context.Users.AddOrUpdateUsers(chunk) - do! context.SaveChangesAsync() |> Task.Ignore + try + for chunk in Seq.chunkBySize 500 users do + use context = new GrinderContext() + do! context.Users.AddOrUpdateUsers(chunk) + do! context.SaveChangesAsync() |> Task.Ignore + with e -> + sprintf "Error on upserting new users %A" users + |> logExn e } |> Async.AwaitTask @@ -52,4 +56,4 @@ open Grinder.Types type IDataAccessApi = abstract member GetUsernameByUserId: TelegramUserId -> Async - abstract member UpsertUsers: User seq -> Async \ No newline at end of file + abstract member UpsertUsers: User seq -> Async \ No newline at end of file diff --git a/src/Grinder/Grinder.fsproj b/src/Grinder/Grinder.fsproj index 2ee0068..f05cd9b 100644 --- a/src/Grinder/Grinder.fsproj +++ b/src/Grinder/Grinder.fsproj @@ -2,16 +2,11 @@ Exe - net5 - false - true - true - true - + @@ -22,9 +17,12 @@ + + + diff --git a/src/Grinder/Log.fs b/src/Grinder/Log.fs new file mode 100644 index 0000000..53fc5ab --- /dev/null +++ b/src/Grinder/Log.fs @@ -0,0 +1,12 @@ +namespace Grinder + +open Serilog + +[] +module Log = + let mutable logger = Unchecked.defaultof + let logInfo (str: string) = logger.Information str + let logExn (exn: exn) (msg: string) = logger.Error(exn, msg) + let logErr (msg: string) = logger.Error msg + let logDbg (str: string) = logger.Debug str + let logFatal (msg: string) (exn: exn) = logger.Fatal(exn, msg) \ No newline at end of file diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index db29908..d1d696f 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -1,7 +1,6 @@ namespace Grinder open System.Net -open System.Threading open Microsoft.Extensions.Configuration open Funogram open Grinder @@ -13,6 +12,7 @@ open Funogram.Bot open Funogram.Types open FunogramExt open System +open Serilog module Program = open System.Net.Http @@ -52,30 +52,69 @@ module Program = let createBotApi config (settings: BotSettings) = { new IBotApi with - member __.DeleteMessage chatId messageId = + member __.DeleteMessage chatId messageId: Async = + sprintf "Deleting message %A in chat %A" messageId chatId + |> logInfo Api.deleteMessage %chatId %messageId |> callApiWithDefaultRetry config - |> Async.Ignore + |> Async.Map (function + | Ok _ -> () // ignoring + | Error apiError -> + sprintf "Error %s with code %d on deleting message %A in chat %A" + apiError.Description + apiError.ErrorCode + messageId + chatId + |> logErr + ) - member __.BanUserByUsername chatUsername userUsername until = + member __.BanUserByUsername chatUsername userUsername until: Async> = + sprintf "Banning user %A by name in chat %A until %A" userUsername chatUsername until + |> logInfo ApiExt.banUserByUsername config %chatUsername %userUsername until - member __.BanUserByUserId chatUsername userId until = + member __.BanUserByUserId chatUsername userId until: Async> = + sprintf "Banning userId %A in chat %A until %A" userId chatUsername until + |> logInfo ApiExt.banUserByUserId config %chatUsername %userId until - member __.UnbanUser chatUsername username = + member __.UnbanUser chatUsername username: Async> = + sprintf "Unbanning user %A in chat %A" username chatUsername + |> logInfo ApiExt.unbanUserByUsername config %chatUsername %username - member __.SendTextMessage chatId text = + member __.SendTextMessage chatId text: Async = + sprintf "Sending {%s} into chat %A" text chatId + |> logInfo ApiExt.sendMessage chatId config text + |> Async.Catch + |> Async.Map (function + | Choice1Of2 () -> () + | Choice2Of2 e -> + sprintf "Error on sending {%s} into chat %A" text chatId + |> logExn e + ) - member __.UnrestrictUser chatUsername username = + member __.UnrestrictUser chatUsername username: Async> = + sprintf "Unrestricting user %A in chat %A"username chatUsername + |> logInfo ApiExt.unrestrictUserByUsername config %chatUsername %username - member __.SendTextToChannel text = + member __.SendTextToChannel text: Async = + sprintf "Sending {%s} into special channel %A" text settings.ChannelId + |> logInfo ApiExt.sendMessage settings.ChannelId config text + |> Async.Catch + |> Async.Map (function + | Choice1Of2 () -> () + | Choice2Of2 e -> + sprintf "Error on sending {%s} into special channel %A" text settings.ChannelId + |> logExn e + ) - member __.PrepareAndDownloadFile fileId = + member __.PrepareAndDownloadFile fileId: Async> = + sprintf "Downloading file %A" fileId + |> logInfo ApiExt.prepareAndDownloadFile config fileId } @@ -86,11 +125,20 @@ module Program = | UsernameFound username -> return Some %(sprintf "@%s" username) | UsernameNotFound -> + sprintf "User %A hasn't been found in Datastore" userId + |> logErr return None } member __.UpsertUsers users = Datastore.upsertUsers users + |> Async.Catch + |> Async.Map (function + | Choice1Of2 () -> () + | Choice2Of2 e -> + sprintf "Error on upserting new users %A" users + |> logExn e + ) } let onUpdate (settings: BotSettings) (botApi: IBotApi) (dataApi: IDataAccessApi) (context: UpdateContext) = @@ -99,10 +147,21 @@ module Program = |> Option.map ^ fun newMessage -> async { match newMessage with | NewAdminPrivateMessage document -> + sprintf "Received: New admin private message with fileId: %s" document.FileId + |> logInfo do! processAdminCommand settings context.Config document.FileId + | NewUsersAdded users -> + sprintf "Received: New users added %A" users + |> logInfo do! processNewUsersCommand users + | NewMessage message -> + sprintf "Received: New message in chat %s from %A " + (defaultArg message.Chat.Title "Unknown") + message.From + |> logInfo + match prepareTextMessage context.Me.Username message with | Some newMessage -> match authorize settings newMessage.FromUsername newMessage.ChatUsername with @@ -111,9 +170,22 @@ module Program = do! command |> Option.map (formatMessage >> botApi.SendTextToChannel) |> Option.defaultValue Async.Unit - | CommandNotAllowed -> () - | None -> () + | CommandNotAllowed -> + sprintf "Command %s NOT allowed for user %A in chat %A" + newMessage.MessageText + newMessage.FromUsername + newMessage.ChatUsername + |> logInfo + | None -> + sprintf "Skipping message %A from %A" message context.Me.Username + |> logDbg + | NewReplyMessage reply -> + sprintf "Received: New reply message in chat %s from %A" + (defaultArg reply.Message.Chat.Title "Unknown") + (reply.Message.From) + |> logInfo + match prepareReplyToMessage context.Me.Username reply with | Some replyMessage -> match authorize settings replyMessage.FromUsername replyMessage.ChatUsername with @@ -122,15 +194,29 @@ module Program = do! command |> Option.map (formatMessage >> botApi.SendTextToChannel) |> Option.defaultValue Async.Unit - | CommandNotAllowed -> () - | None -> () - | IgnoreMessage -> () + | CommandNotAllowed -> + sprintf "Command %s NOT allowed for user %A in chat %A" + replyMessage.MessageText + replyMessage.FromUsername + replyMessage.ChatUsername + |> logInfo + | None -> + sprintf "Skipping message %A from %A" reply context.Me.Username + |> logDbg + | IgnoreMessage -> + sprintf "Ignoring message %A" context.Update.Message + |> logDbg } |> Option.defaultValue Async.Unit } |> Async.Start [] let main _ = + logger <- LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.Console() + .CreateLogger(); + let mutable configBuilder = ConfigurationBuilder() .AddJsonFile("appsettings.json", false, true) @@ -158,8 +244,6 @@ module Program = GrinderContext.MigrateUp() - printfn "Starting bot" - let settings = { Token = config.Token ChatsToMonitor = ChatsToMonitor.Create config.ChatsToMonitor @@ -167,10 +251,20 @@ module Program = ChannelId = %config.ChannelId AdminUserId = %config.AdminUserId } + + string { botConfiguration with Token = "***" } + |> sprintf "Bot Configuration %A" + |> logInfo + + string { settings with Token = "***" } + |> sprintf "Bot Settings %A" + |> logInfo + + logInfo "Starting bot" startBot botConfiguration (onUpdate settings (createBotApi botConfiguration settings) dataApi) None |> Async.Start - printfn "Bot started" + logInfo "Bot started" // Needed for azure web app deploy check. We have to response with anything on port 80 use listener = new HttpListener() @@ -183,7 +277,8 @@ module Program = let ctx = listener.GetContext() let output = ctx.Response.OutputStream output.Write(buffer, 0, buffer.Length) - output.Close(); + output.Close() + logInfo "Sending OK on HTTP request" - printfn "Bot exited" + logInfo "Bot exited" 0 // return an integer exit code \ No newline at end of file diff --git a/tests/Grinder.Tests/Grinder.Tests.fsproj b/tests/Grinder.Tests/Grinder.Tests.fsproj index 24abb3e..87337b2 100644 --- a/tests/Grinder.Tests/Grinder.Tests.fsproj +++ b/tests/Grinder.Tests/Grinder.Tests.fsproj @@ -2,11 +2,8 @@ Exe - net5 false false - true - true @@ -18,7 +15,7 @@ - + From cc1ee4baeae3b09f8fd943275460f883c4bab52b Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Thu, 10 Dec 2020 13:56:39 +0000 Subject: [PATCH 14/15] reduce log noise (#39) Co-authored-by: Ayrat Hudaygulov --- src/Grinder/Program.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index d1d696f..2cfc080 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -157,10 +157,10 @@ module Program = do! processNewUsersCommand users | NewMessage message -> - sprintf "Received: New message in chat %s from %A " + sprintf "Received: New message in chat %s from %s" (defaultArg message.Chat.Title "Unknown") - message.From - |> logInfo + (defaultArg (message.From |> Option.bind(fun x -> x.Username)) "") + |> logDbg match prepareTextMessage context.Me.Username message with | Some newMessage -> @@ -175,7 +175,7 @@ module Program = newMessage.MessageText newMessage.FromUsername newMessage.ChatUsername - |> logInfo + |> logDbg | None -> sprintf "Skipping message %A from %A" message context.Me.Username |> logDbg @@ -199,7 +199,7 @@ module Program = replyMessage.MessageText replyMessage.FromUsername replyMessage.ChatUsername - |> logInfo + |> logDbg | None -> sprintf "Skipping message %A from %A" reply context.Me.Username |> logDbg From 2f52a26dea84989f7e593349dd3770a0bd941fc5 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Thu, 10 Dec 2020 14:09:53 +0000 Subject: [PATCH 15/15] reduce log noise (#40) Co-authored-by: Ayrat Hudaygulov --- src/Grinder/Program.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Grinder/Program.fs b/src/Grinder/Program.fs index 2cfc080..6872f04 100644 --- a/src/Grinder/Program.fs +++ b/src/Grinder/Program.fs @@ -158,7 +158,7 @@ module Program = | NewMessage message -> sprintf "Received: New message in chat %s from %s" - (defaultArg message.Chat.Title "Unknown") + (defaultArg message.Chat.Title "") (defaultArg (message.From |> Option.bind(fun x -> x.Username)) "") |> logDbg @@ -182,9 +182,9 @@ module Program = | NewReplyMessage reply -> sprintf "Received: New reply message in chat %s from %A" - (defaultArg reply.Message.Chat.Title "Unknown") - (reply.Message.From) - |> logInfo + (defaultArg reply.Message.Chat.Title "") + (defaultArg (reply.Message.From |> Option.bind(fun x -> x.Username)) "") + |> logDbg match prepareReplyToMessage context.Me.Username reply with | Some replyMessage ->