From c9d660bf125d55a01157a97a6bef6993cd16d30e Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:58:24 +0900 Subject: [PATCH 01/55] chore(deps): to websockets --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d697330..9cefa98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ iniconfig==2.0.0 itsdangerous==2.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 +Misskey.py==4.1.0 packaging==24.0 pluggy==1.5.0 pyreadline3==3.4.1 @@ -19,5 +20,5 @@ python-dotenv==1.0.1 redis==5.0.5 requests==2.32.3 urllib3==2.2.1 -websocket-client==1.8.0 +websockets==12.0 Werkzeug==3.0.3 From 4f025f5a5f4fb9845c4c0b288602e9e496ce05d0 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:58:50 +0900 Subject: [PATCH 02/55] =?UTF-8?q?feat:=20asyncio=E3=82=92=E4=BD=BF?= =?UTF-8?q?=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index 98ed5c5..224b595 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -4,7 +4,7 @@ from threading import Thread import coloredlogs -import websocket +import websockets import utils import logging_styles @@ -105,16 +105,23 @@ def on_close(self, ws, status_code, msg) -> None: if self._restart: self.start_bot() - def start_bot(self): + async def start_bot(self): streaming_api = f"wss://{misskey.HOST}/streaming?i={misskey.TOKEN}" USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" # NOQA MESSAGE = {"type": "connect", "body": {"channel": "hybridTimeline", "id": "1"}} - # WebSocketの接続 - ws = websocket.WebSocketApp( - streaming_api, - on_message=self.on_message, on_error=self.on_error, on_close=self.on_close, - header={"User-Agent": USER_AGENT} - ) - ws.on_open = lambda ws: ws.send(json.dumps(MESSAGE)) - self.logger.info("Bot was started!") - ws.run_forever() + + async with websockets.connect(streaming_api, user_agent_header=USER_AGENT) as ws: + # self.on_open(ws) + self.logger.info("Bot was started!") + await ws.send(json.dumps(MESSAGE)) + while True: + try: + msg = await ws.recv() + # await self.on_message(ws, msg) + pass + except websockets.ConnectionClosed: + # await self.on_close(ws, ws.close_code, ws.close_reason) + pass + except Exception as e: + # await self.on_error(ws, e) + pass From d36d30bb6b2974f223887425bd93c24566fa3444 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Fri, 7 Jun 2024 23:25:51 +0900 Subject: [PATCH 03/55] =?UTF-8?q?feat:=20=E3=81=A8=E3=82=8A=E3=81=82?= =?UTF-8?q?=E3=81=88=E3=81=9A=E9=96=A2=E6=95=B0=E3=82=92async=20def?= =?UTF-8?q?=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index 224b595..a4cf5c1 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -50,7 +50,7 @@ def need(self) -> bool: return True return False - def on_message(self, ws, message: str) -> None: + async def on_message(self, ws, message: str) -> None: note_body = json.loads(message)["body"]["body"] note_id = note_body["id"] note_text = note_body["text"] @@ -97,31 +97,34 @@ def on_message(self, ws, message: str) -> None: utils.update_db("have_note_user_ids", self.db, False) self.logger.info(f"DataBase Updated. | length: {count}") - def on_error(self, ws, error) -> None: + async def on_error(self, ws, error) -> None: self.logger.warning(str(error)) - def on_close(self, ws, status_code, msg) -> None: + async def on_close(self, ws, status_code, msg) -> bool: self.logger.error(f"WebSocket closed. | code:{status_code} msg: {msg}") - if self._restart: - self.start_bot() + return self._restart async def start_bot(self): streaming_api = f"wss://{misskey.HOST}/streaming?i={misskey.TOKEN}" USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" # NOQA - MESSAGE = {"type": "connect", "body": {"channel": "hybridTimeline", "id": "1"}} - - async with websockets.connect(streaming_api, user_agent_header=USER_AGENT) as ws: - # self.on_open(ws) - self.logger.info("Bot was started!") - await ws.send(json.dumps(MESSAGE)) - while True: - try: - msg = await ws.recv() - # await self.on_message(ws, msg) - pass - except websockets.ConnectionClosed: - # await self.on_close(ws, ws.close_code, ws.close_reason) - pass - except Exception as e: - # await self.on_error(ws, e) - pass + CONNECTMSG = {"type": "connect", "body": {"channel": "hybridTimeline", "id": "1"}} + + while True: + flg = False + async with websockets.connect(streaming_api, user_agent_header=USER_AGENT) as ws: + # self.on_open(ws) + self.logger.info("Bot was started!") + await ws.send(json.dumps(CONNECTMSG)) + while True: + try: + msg = await ws.recv() + await self.on_message(ws, str(msg)) + pass + except websockets.ConnectionClosed: + flg = await self.on_close(ws, ws.close_code, ws.close_reason) + break + except Exception as e: + # await self.on_error(ws, e) + pass + if not flg: + break From 787ed0613bf86c642e6e4367c1e67f85d4a72603 Mon Sep 17 00:00:00 2001 From: fffena <132272963+fffena@users.noreply.github.com> Date: Fri, 7 Jun 2024 23:55:11 +0000 Subject: [PATCH 04/55] lint: --- src/mainbot.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index a4cf5c1..e80ac43 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -64,8 +64,10 @@ async def on_message(self, ws, message: str) -> None: # Renote不可ならreturn return_flg = True if self.ngw.match(note_text): - self.logger.info(f"Detected NG word. | noteId: {note_id}, \ - word: {self.ngw.why(note_text)}") + self.logger.info( + f"Detected NG word. | noteId: {note_id}, \ + word: {self.ngw.why(note_text)}" + ) elif misskey.can_reply(note_body): Thread(target=misskey.reply, args=(note_id, "Pong!")).start() elif not misskey.can_renote(note_body): @@ -107,7 +109,10 @@ async def on_close(self, ws, status_code, msg) -> bool: async def start_bot(self): streaming_api = f"wss://{misskey.HOST}/streaming?i={misskey.TOKEN}" USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" # NOQA - CONNECTMSG = {"type": "connect", "body": {"channel": "hybridTimeline", "id": "1"}} + CONNECTMSG = { + "type": "connect", + "body": {"channel": "hybridTimeline", "id": "1"}, + } while True: flg = False @@ -119,12 +124,10 @@ async def start_bot(self): try: msg = await ws.recv() await self.on_message(ws, str(msg)) - pass except websockets.ConnectionClosed: flg = await self.on_close(ws, ws.close_code, ws.close_reason) break except Exception as e: - # await self.on_error(ws, e) - pass + await self.on_error(ws, e) if not flg: break From 3087f661361f4f3308b29454938d013aaf3017e3 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Sun, 9 Jun 2024 10:31:04 +0900 Subject: [PATCH 05/55] =?UTF-8?q?feat:=20=E5=86=8D=E8=B5=B7=E6=99=82?= =?UTF-8?q?=E3=81=AB=E9=96=93=E9=9A=94=E3=82=92=E9=96=8B=E3=81=91=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mainbot.py b/src/mainbot.py index e80ac43..ee95b06 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import os @@ -131,3 +132,4 @@ async def start_bot(self): await self.on_error(ws, e) if not flg: break + await asyncio.sleep(5) From 5598dbd83ce7b3f84dc0d97f98ad70226489cb14 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Sun, 9 Jun 2024 10:39:36 +0900 Subject: [PATCH 06/55] =?UTF-8?q?chore(deps):=20=E3=81=BE=E3=81=A0?= =?UTF-8?q?=E4=BD=BF=E3=81=A3=E3=81=A6=E3=81=84=E3=81=AA=E3=81=84=E3=83=A9?= =?UTF-8?q?=E3=82=A4=E3=83=96=E3=83=A9=E3=83=AA=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9cefa98..e23465f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ iniconfig==2.0.0 itsdangerous==2.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 -Misskey.py==4.1.0 packaging==24.0 pluggy==1.5.0 pyreadline3==3.4.1 From d04623d849f79612f84e6c7d41c793c74676cad6 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Sun, 9 Jun 2024 10:40:00 +0900 Subject: [PATCH 07/55] chore(deps): pydantic --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index e23465f..2941deb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +annotated-types==0.7.0 blinker==1.8.2 certifi==2024.6.2 charset-normalizer==3.3.2 @@ -13,11 +14,14 @@ Jinja2==3.1.4 MarkupSafe==2.1.5 packaging==24.0 pluggy==1.5.0 +pydantic==2.7.3 +pydantic_core==2.18.4 pyreadline3==3.4.1 pytest==8.2.2 python-dotenv==1.0.1 redis==5.0.5 requests==2.32.3 +typing_extensions==4.12.2 urllib3==2.2.1 websockets==12.0 Werkzeug==3.0.3 From adcd442854c4f454a0450ce005e3d148c78755ac Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Sun, 9 Jun 2024 10:41:53 +0900 Subject: [PATCH 08/55] =?UTF-8?q?chore(deps):=20pydantic=5Fsettings?= =?UTF-8?q?=E3=81=A7=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E3=81=AE=E7=AE=A1?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2941deb..735c7a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ MarkupSafe==2.1.5 packaging==24.0 pluggy==1.5.0 pydantic==2.7.3 +pydantic-settings==2.3.1 pydantic_core==2.18.4 pyreadline3==3.4.1 pytest==8.2.2 From 022d24e0c2b221e5c5694cb3e94d561e5161b271 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Sun, 9 Jun 2024 11:09:58 +0900 Subject: [PATCH 09/55] =?UTF-8?q?feat:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=82=92pydantic=E3=81=A7=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/environs.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/environs.py diff --git a/src/environs.py b/src/environs.py new file mode 100644 index 0000000..5ea8e1f --- /dev/null +++ b/src/environs.py @@ -0,0 +1,19 @@ +from pydantic import RedisDsn +from pydantic_settings import BaseSettings, SettingsConfigDict + +from typing import Literal + + +class DotenvSettings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + +class Settings(DotenvSettings): + host: str + secret_token: str + + db_type: Literal["redis", "pickle"] = "redis" + db_url: RedisDsn + + config_dir: str = "./config" + run_server: bool = False From 634be3372ae8654acda12748e586116f96a467a9 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Sun, 9 Jun 2024 11:18:56 +0900 Subject: [PATCH 10/55] =?UTF-8?q?feat:=20config=5Fdir=E3=82=92=E3=83=90?= =?UTF-8?q?=E3=83=AA=E3=83=87=E3=83=BC=E3=82=B7=E3=83=A7=E3=83=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/environs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/environs.py b/src/environs.py index 5ea8e1f..7051069 100644 --- a/src/environs.py +++ b/src/environs.py @@ -1,8 +1,9 @@ -from pydantic import RedisDsn -from pydantic_settings import BaseSettings, SettingsConfigDict - +from pathlib import Path from typing import Literal +from pydantic import DirectoryPath, RedisDsn +from pydantic_settings import BaseSettings, SettingsConfigDict + class DotenvSettings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") @@ -15,5 +16,5 @@ class Settings(DotenvSettings): db_type: Literal["redis", "pickle"] = "redis" db_url: RedisDsn - config_dir: str = "./config" + config_dir: DirectoryPath = Path("./config") run_server: bool = False From fd8a4b44b6eb12a2aa174ce68719264ea0464bb0 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Sun, 9 Jun 2024 12:10:26 +0900 Subject: [PATCH 11/55] =?UTF-8?q?add:=20=E3=81=A8=E3=82=8A=E3=81=82?= =?UTF-8?q?=E3=81=88=E3=81=9Aschema=E3=82=92=E5=AE=9A=E7=BE=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/userdb.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/userdb.py diff --git a/src/userdb.py b/src/userdb.py new file mode 100644 index 0000000..f7b9536 --- /dev/null +++ b/src/userdb.py @@ -0,0 +1,7 @@ +from redis_om import HashModel, get_redis_connection +from pydantic import PastDatetime + + +class UserInfo(HashModel): + user_name: str + last_recieved_date: PastDatetime From dca050e8ec42461de764ada902ae8ae3c8405e37 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Mon, 10 Jun 2024 20:39:35 +0900 Subject: [PATCH 12/55] =?UTF-8?q?feat:=20=E5=9F=BA=E6=9C=AC=E7=9A=84?= =?UTF-8?q?=E3=81=AAdb=E9=96=A2=E4=BF=82=E3=81=AE=E6=A9=9F=E6=A7=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/userdb.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/userdb.py b/src/userdb.py index f7b9536..0e4a325 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -1,7 +1,44 @@ -from redis_om import HashModel, get_redis_connection +import asyncio + from pydantic import PastDatetime +from aredis_om import HashModel, get_redis_connection, Field + +from datetime import datetime class UserInfo(HashModel): + user_id: str = Field(primary_key=True) user_name: str - last_recieved_date: PastDatetime + last_received_date: PastDatetime + + class Meta: + global_key_prefix = "MisskeyWelcomeBot" + model_key_prefix = "PostedUsers" + + +class UserDB: + def __init__(self, redis_url: str) -> None: + self._db_url = redis_url + UserInfo.Meta.database = get_redis_connection(url=self._db_url, decode_responses=True) # type: ignore + + async def get_all_users(self) -> list[UserInfo]: + all_pks = await UserInfo.all_pks() + return await asyncio.gather(*[UserInfo.get(i) async for i in all_pks]) + + async def get_user(self, user_id: str) -> UserInfo: + return await UserInfo.get(user_id) + + async def add_user(self, user_id: str, username: str): + await UserInfo( + user_id=user_id, + user_name=username, + last_received_date=datetime.now() + ).save() + + +if __name__ == "__main__": + import asyncio + from environs import Settings + + db = UserDB(str(Settings().db_url)) + print(asyncio.run(db.get_all_users())) From 760813caa2fc9419afbc4582779ad1f5a94aabdd Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:48:38 +0900 Subject: [PATCH 13/55] =?UTF-8?q?feat:=20=E6=A4=9C=E7=B4=A2=E6=A9=9F?= =?UTF-8?q?=E8=83=BD=E3=81=AE=E8=BF=BD=E5=8A=A0=E3=81=A8=E3=83=A6=E3=83=BC?= =?UTF-8?q?=E3=82=B6=E3=83=BC=E5=90=8D=E3=82=92=E4=BF=9D=E5=AD=98=E3=81=99?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/userdb.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/userdb.py b/src/userdb.py index 0e4a325..b3347eb 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -1,14 +1,14 @@ import asyncio from pydantic import PastDatetime -from aredis_om import HashModel, get_redis_connection, Field +from aredis_om import HashModel, Migrator, get_redis_connection, Field, NotFoundError from datetime import datetime class UserInfo(HashModel): user_id: str = Field(primary_key=True) - user_name: str + user_name: str = Field(index=True) last_received_date: PastDatetime class Meta: @@ -19,26 +19,38 @@ class Meta: class UserDB: def __init__(self, redis_url: str) -> None: self._db_url = redis_url - UserInfo.Meta.database = get_redis_connection(url=self._db_url, decode_responses=True) # type: ignore + UserInfo.Meta.database = get_redis_connection(url=self._db_url) # type: ignore async def get_all_users(self) -> list[UserInfo]: all_pks = await UserInfo.all_pks() return await asyncio.gather(*[UserInfo.get(i) async for i in all_pks]) - async def get_user(self, user_id: str) -> UserInfo: - return await UserInfo.get(user_id) + async def get_user_by_id(self, user_id: str) -> UserInfo | None: + try: + return await UserInfo.get(user_id) + except NotFoundError: + return None + + async def get_user_by_name(self, aas: str) -> UserInfo | None: + await self._migrate() + a = UserInfo.find(UserInfo.user_name == aas) + try: + return await a.first() # type: ignore + except NotFoundError: + return None async def add_user(self, user_id: str, username: str): await UserInfo( - user_id=user_id, - user_name=username, - last_received_date=datetime.now() + user_id=user_id, user_name=username, last_received_date=datetime.now() ).save() + async def _migrate(self): + await Migrator().run() + if __name__ == "__main__": import asyncio - from environs import Settings - db = UserDB(str(Settings().db_url)) - print(asyncio.run(db.get_all_users())) + DB_URL = "redis://localhost:6379" + db = UserDB(DB_URL) + print(type(asyncio.run(db.get_user_by_name("ffdi")))) From b1e920df55efbedccb3b1942ed2c440af0c33ccd Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:48:45 +0900 Subject: [PATCH 14/55] =?UTF-8?q?chore:=20=E5=9E=8B=E3=83=92=E3=83=B3?= =?UTF-8?q?=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/userdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/userdb.py b/src/userdb.py index b3347eb..5276921 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -39,12 +39,12 @@ async def get_user_by_name(self, aas: str) -> UserInfo | None: except NotFoundError: return None - async def add_user(self, user_id: str, username: str): + async def add_user(self, user_id: str, username: str) -> None: await UserInfo( user_id=user_id, user_name=username, last_received_date=datetime.now() ).save() - async def _migrate(self): + async def _migrate(self) -> None: await Migrator().run() From 0e008f27c36fe917073be079ee5c37d390e1e702 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:04:02 +0900 Subject: [PATCH 15/55] =?UTF-8?q?fix:=20Pylance=E3=81=AB=E6=96=87=E5=8F=A5?= =?UTF-8?q?=E3=82=92=E8=A8=80=E3=82=8F=E3=82=8C=E3=81=AA=E3=81=84=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=20from:=20https://github.com/pydantic/pydant?= =?UTF-8?q?ic/issues/3753?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/environs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/environs.py b/src/environs.py index 7051069..beef726 100644 --- a/src/environs.py +++ b/src/environs.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Literal -from pydantic import DirectoryPath, RedisDsn +from pydantic import DirectoryPath, RedisDsn, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -10,11 +10,11 @@ class DotenvSettings(BaseSettings): class Settings(DotenvSettings): - host: str - secret_token: str + host: str = Field(default=...) + secret_token: str = Field(default=...) db_type: Literal["redis", "pickle"] = "redis" - db_url: RedisDsn + db_url: RedisDsn = Field(default=...) config_dir: DirectoryPath = Path("./config") run_server: bool = False From 306e31f6a87209feed6f8eb9399b80c544896339 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:39:55 +0900 Subject: [PATCH 16/55] =?UTF-8?q?fix:=20redis=E4=BB=A5=E5=A4=96=E3=81=AE?= =?UTF-8?q?=E3=81=A8=E3=81=8D=E3=81=AB=E3=81=AFdb=5Furl=E3=81=8C=E3=81=84?= =?UTF-8?q?=E3=82=89=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/environs.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/environs.py b/src/environs.py index beef726..a077425 100644 --- a/src/environs.py +++ b/src/environs.py @@ -1,7 +1,7 @@ from pathlib import Path -from typing import Literal +from typing import Literal, Optional -from pydantic import DirectoryPath, RedisDsn, Field +from pydantic import DirectoryPath, RedisDsn, Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -14,7 +14,13 @@ class Settings(DotenvSettings): secret_token: str = Field(default=...) db_type: Literal["redis", "pickle"] = "redis" - db_url: RedisDsn = Field(default=...) + db_url: Optional[RedisDsn] = None config_dir: DirectoryPath = Path("./config") run_server: bool = False + + @model_validator(mode="after") + def required_when_redis(self): + if self.db_url is None and self.db_type == "redis": + raise ValueError("DB_URL is required when DB_TYPE is redis") + return self From dc1c0cf8aa3ff34234b969bff686ff3e40b9cb36 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:41:53 +0900 Subject: [PATCH 17/55] =?UTF-8?q?refactor:=20=E5=A4=89=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=81=AA=E3=81=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/userdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/userdb.py b/src/userdb.py index 5276921..8f6cd52 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -33,15 +33,17 @@ async def get_user_by_id(self, user_id: str) -> UserInfo | None: async def get_user_by_name(self, aas: str) -> UserInfo | None: await self._migrate() - a = UserInfo.find(UserInfo.user_name == aas) + found = UserInfo.find(UserInfo.user_name == aas) try: - return await a.first() # type: ignore + return await found.first() # type: ignore except NotFoundError: return None async def add_user(self, user_id: str, username: str) -> None: await UserInfo( - user_id=user_id, user_name=username, last_received_date=datetime.now() + user_id=user_id, + user_name=username, + last_received_date=datetime.now() ).save() async def _migrate(self) -> None: From 5f52b48e44ac9c0564535c4669ceaa955e2ad287 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:44:29 +0900 Subject: [PATCH 18/55] lint --- src/userdb.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/userdb.py b/src/userdb.py index 8f6cd52..85fd993 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -1,9 +1,14 @@ import asyncio +from datetime import datetime from pydantic import PastDatetime -from aredis_om import HashModel, Migrator, get_redis_connection, Field, NotFoundError - -from datetime import datetime +from aredis_om import ( + HashModel, + Migrator, + get_redis_connection, + Field, + NotFoundError, +) class UserInfo(HashModel): From fd7726b3b649c80f1ae2fa78ae0e75fd3c4e4485 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:45:12 +0900 Subject: [PATCH 19/55] =?UTF-8?q?fix:=20=E4=BD=99=E5=88=86=E3=81=AA?= =?UTF-8?q?=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E3=82=92=E7=84=A1=E8=A6=96?= =?UTF-8?q?=E3=81=97=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/environs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/environs.py b/src/environs.py index a077425..accf3bb 100644 --- a/src/environs.py +++ b/src/environs.py @@ -6,7 +6,7 @@ class DotenvSettings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") class Settings(DotenvSettings): From 7fd7c88823d3b9e3d9e067aac44bf6f48f2b2452 Mon Sep 17 00:00:00 2001 From: fffena <132272963+fffena@users.noreply.github.com> Date: Tue, 11 Jun 2024 07:36:40 +0000 Subject: [PATCH 20/55] =?UTF-8?q?chore(deps):=20redis-om=E3=81=AE=E5=85=A5?= =?UTF-8?q?=E3=82=8C=E5=BF=98=E3=82=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/requirements.txt b/requirements.txt index 735c7a4..c36f3ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,38 @@ annotated-types==0.7.0 blinker==1.8.2 certifi==2024.6.2 +cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 colorama==0.4.6 coloredlogs==15.0.1 +cryptography==42.0.8 Flask==3.0.3 +hiredis==2.3.2 humanfriendly==10.0 idna==3.7 iniconfig==2.0.0 itsdangerous==2.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 +more-itertools==10.3.0 packaging==24.0 pluggy==1.5.0 +pycparser==2.22 pydantic==2.7.3 pydantic-settings==2.3.1 pydantic_core==2.18.4 pyreadline3==3.4.1 pytest==8.2.2 python-dotenv==1.0.1 +python-ulid==1.1.0 redis==5.0.5 +redis-om==0.3.1 requests==2.32.3 +types-cffi==1.16.0.20240331 +types-pyOpenSSL==24.1.0.20240425 +types-redis==4.6.0.20240425 +types-setuptools==70.0.0.20240524 typing_extensions==4.12.2 urllib3==2.2.1 websockets==12.0 From 068d8844f8d5f516414762d80706ac4c51b02123 Mon Sep 17 00:00:00 2001 From: fffena <132272963+fffena@users.noreply.github.com> Date: Tue, 11 Jun 2024 07:58:23 +0000 Subject: [PATCH 21/55] =?UTF-8?q?feat(wip):=20environs.py=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index ee95b06..a833b4b 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -1,7 +1,6 @@ import asyncio import json import logging -import os from threading import Thread import coloredlogs @@ -10,6 +9,8 @@ import utils import logging_styles import misskey_api as misskey +from environs import Settings +from userdb import UserDB from ngwords import NGWords from emojis import EmojiSet @@ -17,21 +18,22 @@ class Bot: counter = utils.Counter(100, lambda: None) - def __init__(self, restart: bool = True) -> None: + def __init__(self, settings: Settings, restart: bool = True) -> None: logger = logging.getLogger(__name__) logging_styles.set_default() coloredlogs.install(logger=logger) + self.config = settings self.logger = logger self._restart = restart - self.config_dir = utils.config_dir() + self.config_dir = self.config.config_dir logger.info("Loading response.json...") - self.emojis = EmojiSet(os.path.join(self.config_dir, "response.json")) + self.emojis = EmojiSet(str(self.config_dir.joinpath("response.json"))) logger.info("Loading ngwords.txt...") - self.ngw = NGWords(os.path.join(self.config_dir, "ngwords.txt")) + self.ngw = NGWords(str(self.config_dir.joinpath("ngwords.txt"))) - self.db = utils.get_db() + self.db = UserDB(str(self.config.db_url)) # TODO: redis以外への対応 # TODO: なんか良い名前に変えたい def send_welcome(self, note_id: str, note_text: str) -> None: From 5b54ca68537c2871b86ee8ce6a1bf3baa3896545 Mon Sep 17 00:00:00 2001 From: fffena <132272963+fffena@users.noreply.github.com> Date: Tue, 11 Jun 2024 08:29:09 +0000 Subject: [PATCH 22/55] =?UTF-8?q?fix:=20UserDB=E3=82=AF=E3=83=A9=E3=82=B9?= =?UTF-8?q?=E3=82=92=E4=BD=BF=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index a833b4b..1036de8 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -75,7 +75,7 @@ async def on_message(self, ws, message: str) -> None: Thread(target=misskey.reply, args=(note_id, "Pong!")).start() elif not misskey.can_renote(note_body): pass - elif note_body["userId"] in set(self.db): + elif self.db.get_user_by_id(note_body["userId"]) is None: self.logger.debug("Skiped api request because it was registered in DB.") else: return_flg = False @@ -97,7 +97,7 @@ async def on_message(self, ws, message: str) -> None: return None if notes_count > 5: - self.db.append(note_body["userId"]) + self.db.add_user(note_body["userId"], note_body["user"]["name"]) if (count := len(self.db)) % 100 == 0: utils.update_db("have_note_user_ids", self.db, False) self.logger.info(f"DataBase Updated. | length: {count}") From 8d0f0e1ba2a0b10709a42b34b1c5a2843d46df61 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:00:52 +0900 Subject: [PATCH 23/55] =?UTF-8?q?fix:=20getLogger()=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E3=81=97=E3=80=81=E3=82=B0=E3=83=AD=E3=83=BC=E3=83=90?= =?UTF-8?q?=E3=83=AB=E5=A4=89=E6=95=B0=E3=81=AE=E6=9B=B8=E3=81=8D=E6=8F=9B?= =?UTF-8?q?=E3=81=88=E3=82=92=E6=B6=88=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/keep_alive.py | 6 +----- src/logging_styles.py | 19 +++++++++++++------ src/mainbot.py | 11 +++-------- src/misskey_api.py | 6 +----- 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/keep_alive.py b/src/keep_alive.py index 6b2f113..88d46e9 100644 --- a/src/keep_alive.py +++ b/src/keep_alive.py @@ -1,8 +1,6 @@ -import logging from multiprocessing import Process from os import getenv -import coloredlogs from dotenv import load_dotenv from flask import Flask, jsonify @@ -12,9 +10,7 @@ app = Flask("app") -logger = logging.getLogger(__name__) -logging_styles.set_default() -coloredlogs.install(logger=logger) +logger = logging_styles.getLogger(__name__) @app.get("/") diff --git a/src/logging_styles.py b/src/logging_styles.py index 4ff1ca3..93ab711 100644 --- a/src/logging_styles.py +++ b/src/logging_styles.py @@ -19,10 +19,17 @@ "critical": {"color": "red"}, } +formatter = coloredlogs.ColoredFormatter( + fmt=DEFAULT_LOG_FORMAT, + datefmt=DEFAULT_DATE_FORMAT, + field_styles=DEFAULT_FIELD_STYLES, + level_styles=DEFAULT_LEVEL_STYLES, +) -def set_default(): - coloredlogs.DEFAULT_LOG_LEVEL = DEFAULT_LOG_LEVEL - coloredlogs.DEFAULT_LOG_FORMAT = DEFAULT_LOG_FORMAT - coloredlogs.DEFAULT_DATE_FORMAT = DEFAULT_DATE_FORMAT - coloredlogs.DEFAULT_FIELD_STYLES = DEFAULT_FIELD_STYLES - coloredlogs.DEFAULT_LEVEL_STYLES = DEFAULT_LEVEL_STYLES + +def getLogger(name: str) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(DEFAULT_LOG_LEVEL) + logger.addHandler(logging.StreamHandler()) + logger.handlers[0].setFormatter(formatter) + return logger diff --git a/src/mainbot.py b/src/mainbot.py index 1036de8..1ad1627 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -1,9 +1,7 @@ import asyncio import json -import logging from threading import Thread -import coloredlogs import websockets import utils @@ -19,18 +17,15 @@ class Bot: counter = utils.Counter(100, lambda: None) def __init__(self, settings: Settings, restart: bool = True) -> None: - logger = logging.getLogger(__name__) - logging_styles.set_default() - coloredlogs.install(logger=logger) + self.logger = logging_styles.getLogger(__name__) self.config = settings - self.logger = logger self._restart = restart self.config_dir = self.config.config_dir - logger.info("Loading response.json...") + self.logger.info("Loading response.json...") self.emojis = EmojiSet(str(self.config_dir.joinpath("response.json"))) - logger.info("Loading ngwords.txt...") + self.logger.info("Loading ngwords.txt...") self.ngw = NGWords(str(self.config_dir.joinpath("ngwords.txt"))) self.db = UserDB(str(self.config.db_url)) # TODO: redis以外への対応 diff --git a/src/misskey_api.py b/src/misskey_api.py index e676a66..5dce152 100644 --- a/src/misskey_api.py +++ b/src/misskey_api.py @@ -1,10 +1,8 @@ import pickle -import logging from os import getenv from collections import deque import requests -import coloredlogs from dotenv import load_dotenv from requests import Timeout @@ -17,9 +15,7 @@ TOKEN = getenv("SECRET_TOKEN") USERNAME = requests.post(f"https://{HOST}/api/i", json={"i": TOKEN}).json()["username"] -logger = logging.getLogger(__name__) -logging_styles.set_default() -coloredlogs.install(logger=logger) +logger = logging_styles.getLogger(__name__) limiter = RateLimiter(0.5) limiter2 = RateLimiter(0.5) From f034442df8e583a2733c2e9b951010243eea315b Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:12:53 +0900 Subject: [PATCH 24/55] =?UTF-8?q?fix:=20bot=E8=B5=B7=E5=8B=95=E3=82=92?= =?UTF-8?q?=E7=B0=A1=E7=95=A5=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index 1ad1627..1a1757e 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -113,7 +113,6 @@ async def start_bot(self): } while True: - flg = False async with websockets.connect(streaming_api, user_agent_header=USER_AGENT) as ws: # self.on_open(ws) self.logger.info("Bot was started!") @@ -123,10 +122,10 @@ async def start_bot(self): msg = await ws.recv() await self.on_message(ws, str(msg)) except websockets.ConnectionClosed: - flg = await self.on_close(ws, ws.close_code, ws.close_reason) + await self.on_close(ws, ws.close_code, ws.close_reason) + if not self._restart: + return break except Exception as e: await self.on_error(ws, e) - if not flg: - break await asyncio.sleep(5) From b05cc5ee4fef0c2504f0f0acf6565d67c6edba45 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:18:30 +0900 Subject: [PATCH 25/55] =?UTF-8?q?fix:=20=E9=9D=9E=E5=90=8C=E6=9C=9F?= =?UTF-8?q?=E5=91=BC=E3=81=B3=E5=87=BA=E3=81=97=E3=81=8C=E3=81=A7=E3=81=8D?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index 1a1757e..6775045 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -70,7 +70,7 @@ async def on_message(self, ws, message: str) -> None: Thread(target=misskey.reply, args=(note_id, "Pong!")).start() elif not misskey.can_renote(note_body): pass - elif self.db.get_user_by_id(note_body["userId"]) is None: + elif await self.db.get_user_by_id(note_body["userId"]) is None: self.logger.debug("Skiped api request because it was registered in DB.") else: return_flg = False @@ -92,10 +92,8 @@ async def on_message(self, ws, message: str) -> None: return None if notes_count > 5: - self.db.add_user(note_body["userId"], note_body["user"]["name"]) - if (count := len(self.db)) % 100 == 0: - utils.update_db("have_note_user_ids", self.db, False) - self.logger.info(f"DataBase Updated. | length: {count}") + await self.db.add_user(note_body["userId"], note_body["user"]["name"]) + self.logger.info(f"DataBase Updated.") async def on_error(self, ws, error) -> None: self.logger.warning(str(error)) From 12322f70616d4cd930488106d4896c7fd1c3d75d Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:35:06 +0900 Subject: [PATCH 26/55] =?UTF-8?q?feat:=20bot=E8=B5=B7=E5=8B=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/keep_alive.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/keep_alive.py b/src/keep_alive.py index 88d46e9..86acf1d 100644 --- a/src/keep_alive.py +++ b/src/keep_alive.py @@ -1,11 +1,11 @@ +import asyncio from multiprocessing import Process -from os import getenv -from dotenv import load_dotenv from flask import Flask, jsonify import logging_styles import mainbot +import environs app = Flask("app") @@ -23,8 +23,8 @@ def run_server(): if __name__ == "__main__": - load_dotenv() - if getenv("RUN_SERVER", False): + config = environs.Settings() + if config.run_server: Process(target=run_server).start() logger.info("Web server started!") - mainbot.Bot().start_bot() + asyncio.run(mainbot.Bot(config).start_bot()) From 14bada260d8481df191602eb55ba0e5a38430d71 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:35:27 +0900 Subject: [PATCH 27/55] =?UTF-8?q?fix:=20=E3=83=A6=E3=83=BC=E3=82=B6?= =?UTF-8?q?=E3=83=BC=E5=90=8D=E3=81=AE=E5=8F=96=E5=BE=97=E3=81=8C=E3=81=A7?= =?UTF-8?q?=E3=81=8D=E3=81=A6=E3=81=84=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index 6775045..2f01cb7 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -70,7 +70,7 @@ async def on_message(self, ws, message: str) -> None: Thread(target=misskey.reply, args=(note_id, "Pong!")).start() elif not misskey.can_renote(note_body): pass - elif await self.db.get_user_by_id(note_body["userId"]) is None: + elif await self.db.get_user_by_id(note_body["userId"]): self.logger.debug("Skiped api request because it was registered in DB.") else: return_flg = False @@ -92,7 +92,7 @@ async def on_message(self, ws, message: str) -> None: return None if notes_count > 5: - await self.db.add_user(note_body["userId"], note_body["user"]["name"]) + await self.db.add_user(note_body["userId"], note_body["user"]["username"]) self.logger.info(f"DataBase Updated.") async def on_error(self, ws, error) -> None: From 699d40eb6cc5b714f25695f5ad8a9096b1f381bb Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:43:00 +0900 Subject: [PATCH 28/55] chore: ... --- src/mainbot.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/mainbot.py b/src/mainbot.py index 2f01cb7..fc37f95 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -52,6 +52,7 @@ async def on_message(self, ws, message: str) -> None: note_body = json.loads(message)["body"]["body"] note_id = note_body["id"] note_text = note_body["text"] + user_id = note_body["userId"] if note_text is None: note_text = "" @@ -70,7 +71,7 @@ async def on_message(self, ws, message: str) -> None: Thread(target=misskey.reply, args=(note_id, "Pong!")).start() elif not misskey.can_renote(note_body): pass - elif await self.db.get_user_by_id(note_body["userId"]): + elif await self.db.get_user_by_id(user_id): self.logger.debug("Skiped api request because it was registered in DB.") else: return_flg = False @@ -81,19 +82,19 @@ async def on_message(self, ws, message: str) -> None: self.logger.debug( f"Notes not registered in database. | body: {note_text} , id: {note_id}" ) - user_info = misskey.get_user_info(user_id=note_body["userId"]) + user_info = misskey.get_user_info(user_id=user_id) if (notes_count := user_info["notesCount"]) == 1: self.send_welcome(note_id, note_text) elif notes_count <= 10: # ノート数が10以下ならRenote出来る可能性 - notes = misskey.get_user_notes(note_body["userId"], note_id, 10) + notes = misskey.get_user_notes(user_id, note_id, 10) if all([not misskey.can_renote(note) for note in notes]): self.send_welcome(note_id, note_text) return None if notes_count > 5: - await self.db.add_user(note_body["userId"], note_body["user"]["username"]) - self.logger.info(f"DataBase Updated.") + await self.db.add_user(user_id, note_body["user"]["username"]) + self.logger.info("DataBase Updated.") async def on_error(self, ws, error) -> None: self.logger.warning(str(error)) From 3ff1ab5aecaed3eea6c1da56ca5949a8f4f28f43 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:44:32 +0900 Subject: [PATCH 29/55] =?UTF-8?q?delete:=20=E5=BD=B9=E7=9B=AE=E3=82=92?= =?UTF-8?q?=E7=B5=82=E3=81=88=E3=81=9Futils=E3=81=AE=E9=96=A2=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils.py | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/src/utils.py b/src/utils.py index 9b6f5f1..f7a8758 100644 --- a/src/utils.py +++ b/src/utils.py @@ -47,50 +47,3 @@ def wrapper(*args, **kwargs): resp = f(*args, **kwargs) return resp return wrapper - - -def config_dir(): - dotenv.load_dotenv() - dir = os.getenv("CONFIG_DIR", "./config") - if not os.path.exists(dir): - raise FileNotFoundError("Config directory not found.") - return dir - - -def db_type(): - dotenv.load_dotenv() - return os.getenv("DB_TYPE") - - -def update_db(key: str, value, allow_duplicates: bool = True) -> None: - if not allow_duplicates: - value = set(value) - - if db_type() == "redis": - import redis - dotenv.load_dotenv() - - r = redis.from_url(os.getenv("DB_URL")) - p = r.pipeline() - for i in value: - p.sadd("have_note_user_ids", i) - p.execute() - elif db_type() == "pickle": - with open("./data/users.pickle", "wb") as f: - pickle.dump(deque(value), f) - - -def get_db(): - if db_type() == "redis": - import redis - dotenv.load_dotenv() - - r = redis.from_url(os.getenv("DB_URL")) - return deque(map(lambda x: x.decode(), r.smembers("have_note_user_ids"))) - elif db_type() == "pickle": - try: - with open('./data/users.pickle', "rb") as f: - have_note_user_ids = pickle.load(f) - except FileNotFoundError: - have_note_user_ids = deque() - return have_note_user_ids From 36cb687da07596c0f5273a7dbfa65688c8fc719f Mon Sep 17 00:00:00 2001 From: fffena <132272963+fffena@users.noreply.github.com> Date: Wed, 12 Jun 2024 08:29:12 +0000 Subject: [PATCH 30/55] =?UTF-8?q?chore:=20cSpell=E3=81=AE=E9=99=A4?= =?UTF-8?q?=E5=A4=96=E3=83=AF=E3=83=BC=E3=83=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/settings.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a5c31cd..98af5f9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,11 @@ { "cSpell.words": [ - "dotenv" + "coloredlogs", + "dotenv", + "levelname", + "misskey", + "renote", + "websockets" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true From 6dab209f0a3a633b3ad9d20f3df0ae3fa748ce2f Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:32:12 +0900 Subject: [PATCH 31/55] =?UTF-8?q?fix:=20=E4=B8=8D=E5=BF=85=E8=A6=81?= =?UTF-8?q?=E3=81=AAimport=E3=81=AE=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misskey_api.py | 2 -- src/utils.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/src/misskey_api.py b/src/misskey_api.py index 5dce152..1f64c70 100644 --- a/src/misskey_api.py +++ b/src/misskey_api.py @@ -1,6 +1,4 @@ -import pickle from os import getenv -from collections import deque import requests from dotenv import load_dotenv diff --git a/src/utils.py b/src/utils.py index f7a8758..589112d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,10 +1,5 @@ import time -import os -import pickle from typing import Any -from collections import deque - -import dotenv class RateLimiter: From ccb87ccb2a96ddd294a763d55e1cb0a9a1660e84 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:45:17 +0900 Subject: [PATCH 32/55] =?UTF-8?q?fix:=20=E5=9E=8B=E9=96=A2=E9=80=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ngwords.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngwords.py b/src/ngwords.py index b23e5b2..54335c8 100644 --- a/src/ngwords.py +++ b/src/ngwords.py @@ -11,7 +11,7 @@ def __init__(self, path) -> None: self._path = pathlib.Path(path) self._load() - def __getitem__(self, key) -> dict: + def __getitem__(self, key) -> set: if (key := key.lower()) == "ng": return self._ng elif key == "excluded": From 6a3f1c39eff0377e4083b8afa1cbb1b6ef878ccc Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:47:29 +0900 Subject: [PATCH 33/55] lint: --- src/emojis.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/emojis.py b/src/emojis.py index 75fac96..2f49fad 100644 --- a/src/emojis.py +++ b/src/emojis.py @@ -21,10 +21,14 @@ def __init__(self, data: str | dict) -> None: def _check_format(self, json: Any) -> None: if not isinstance(json, dict) or sorted(json.keys()) != ["others", "triggers"]: - raise ConfigJsonError("response.jsonは{'triggers': [], 'others': []}の形にしてください。") + raise ConfigJsonError( + "response.jsonは{'triggers': [], 'others': []}の形にしてください。" + ) if any([tuple(i.keys()) != ("keywords", "emoji") for i in json["triggers"]]): - raise ConfigJsonError("response.jsonのトリガーのキーはkeywordsとemojiにしてください。") + raise ConfigJsonError( + "response.jsonのトリガーのキーはkeywordsとemojiにしてください。" + ) def get_response_emoji(self, text: str) -> str: for i in self.response_emojis: From c45b85ac4da89407ef0289601fbc155347b1ed79 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Wed, 12 Jun 2024 18:02:10 +0900 Subject: [PATCH 34/55] =?UTF-8?q?fix:=20=E5=85=A8=E7=84=B6Any=E3=81=A7?= =?UTF-8?q?=E3=81=AF=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/emojis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/emojis.py b/src/emojis.py index 2f49fad..0148798 100644 --- a/src/emojis.py +++ b/src/emojis.py @@ -1,6 +1,5 @@ import json import random -from typing import Any class ConfigJsonError(Exception): @@ -19,7 +18,7 @@ def __init__(self, data: str | dict) -> None: self.response_emojis = loaded["triggers"] self.others = loaded["others"] - def _check_format(self, json: Any) -> None: + def _check_format(self, json: dict) -> None: if not isinstance(json, dict) or sorted(json.keys()) != ["others", "triggers"]: raise ConfigJsonError( "response.jsonは{'triggers': [], 'others': []}の形にしてください。" From 314e523e3e5a751069364336c13b58f2608f3da4 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Wed, 12 Jun 2024 18:07:23 +0900 Subject: [PATCH 35/55] =?UTF-8?q?fix:=20json=E3=81=AE=E3=82=AD=E3=83=BC?= =?UTF-8?q?=E5=88=A4=E5=AE=9A=E3=81=8C=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84?= =?UTF-8?q?=E5=A0=B4=E5=90=88=E3=81=8C=E3=81=82=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/emojis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emojis.py b/src/emojis.py index 0148798..e44d93e 100644 --- a/src/emojis.py +++ b/src/emojis.py @@ -24,7 +24,7 @@ def _check_format(self, json: dict) -> None: "response.jsonは{'triggers': [], 'others': []}の形にしてください。" ) - if any([tuple(i.keys()) != ("keywords", "emoji") for i in json["triggers"]]): + if any([sorted(i.keys()) != ("emoji", "keywords") for i in json["triggers"]]): raise ConfigJsonError( "response.jsonのトリガーのキーはkeywordsとemojiにしてください。" ) From 8fdebef72ffddf783ca275616c64efb11a8e13d5 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:39:53 +0900 Subject: [PATCH 36/55] =?UTF-8?q?feat:=20path=E3=82=AA=E3=83=96=E3=82=B8?= =?UTF-8?q?=E3=82=A7=E3=82=AF=E3=83=88=E3=82=84string=E3=81=8B=E3=82=89?= =?UTF-8?q?=E8=AA=AD=E3=81=BF=E8=BE=BC=E3=82=80=E9=96=A2=E6=95=B0=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/emojis.py | 9 +++------ src/ngwords.py | 11 ++++++----- src/utils.py | 25 +++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/emojis.py b/src/emojis.py index e44d93e..ed4d2bf 100644 --- a/src/emojis.py +++ b/src/emojis.py @@ -1,6 +1,7 @@ -import json import random +from utils import load_from_json_path + class ConfigJsonError(Exception): pass @@ -8,11 +9,7 @@ class ConfigJsonError(Exception): class EmojiSet: def __init__(self, data: str | dict) -> None: - if isinstance(data, str): - with open(data) as f: - loaded = json.load(f) - else: - loaded = data + loaded = load_from_json_path(data, dict) self._check_format(loaded) self.response_emojis = loaded["triggers"] diff --git a/src/ngwords.py b/src/ngwords.py index 54335c8..ebbec9f 100644 --- a/src/ngwords.py +++ b/src/ngwords.py @@ -1,4 +1,6 @@ -import pathlib +import os + +from utils import load_from_path class NGWords: @@ -7,8 +9,8 @@ class NGWords: initにngワードのテキストファイルのパスを渡してください。 """ - def __init__(self, path) -> None: - self._path = pathlib.Path(path) + def __init__(self, path: str | os.PathLike) -> None: + self.raw = load_from_path(path) self._load() def __getitem__(self, key) -> set: @@ -20,8 +22,7 @@ def __getitem__(self, key) -> set: raise KeyError('"ng"か"excluded"を指定してください') def _load(self) -> None: - with self._path.open() as f: - data = f.read().split("\n") + data = self.raw.split("\n") data = [j.lower() for j in data if j != ""] self._ng = {j for j in data if (j[0] != "-") and (j[0] != "#")} self._allow = {j[1:] for j in data if j[0] == "-"} diff --git a/src/utils.py b/src/utils.py index 589112d..2ad392e 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,5 +1,11 @@ import time -from typing import Any +from typing import Any, TypeVar, Type + +import json +from os import PathLike + + +T = TypeVar("T") class RateLimiter: @@ -33,7 +39,7 @@ def __init__(self, counter, do) -> None: self.do = do def __call__(self, f) -> Any: - def wrapper(*args, **kwargs): + def wrapper(*args: f.args, **kwargs): self._now += 1 if self._now == self.count: self._now = 0 @@ -42,3 +48,18 @@ def wrapper(*args, **kwargs): resp = f(*args, **kwargs) return resp return wrapper + +def load_from_path(path: str | PathLike | T, extend: Type[T] = Type[Any]) -> str | T: + if not isinstance(path, (str, PathLike, extend)): + raise TypeError(f"Invalid type for path: {type(path)}. Expected str, PathLike, or {extend.__name__}.") + + if isinstance(path, extend): + return path + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def load_from_json_path(path: str | PathLike | T, extend: Type[T] = Type[Any]) -> dict | T: + if isinstance(path, extend): + return path + return json.loads(load_from_path(path)) From 145bc652e3bf1079cf08794cc413887a37ff0922 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:53:10 +0900 Subject: [PATCH 37/55] =?UTF-8?q?fix:=20=E6=B5=81=E7=9F=B3=E3=81=ABType[An?= =?UTF-8?q?y]=E3=81=AF=E3=83=80=E3=83=A1=E3=81=A0=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils.py b/src/utils.py index 2ad392e..af050ab 100644 --- a/src/utils.py +++ b/src/utils.py @@ -49,17 +49,18 @@ def wrapper(*args: f.args, **kwargs): return resp return wrapper -def load_from_path(path: str | PathLike | T, extend: Type[T] = Type[Any]) -> str | T: + +def load_from_path(path: str | PathLike | T, extend: Type[T | None] = type(None)) -> str | T: if not isinstance(path, (str, PathLike, extend)): raise TypeError(f"Invalid type for path: {type(path)}. Expected str, PathLike, or {extend.__name__}.") - if isinstance(path, extend): + if extend is not None and isinstance(path, extend): return path with open(path, "r", encoding="utf-8") as f: return f.read() -def load_from_json_path(path: str | PathLike | T, extend: Type[T] = Type[Any]) -> dict | T: +def load_from_json_path(path: str | PathLike | T, extend: Type[T | None] = type(None)) -> dict | T: if isinstance(path, extend): return path return json.loads(load_from_path(path)) From b3d7e0c7cb330430a65c593bcd717af6f431175c Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:13:40 +0900 Subject: [PATCH 38/55] =?UTF-8?q?test:=20pytest=E3=81=A7=E7=9B=B4=E6=8E=A5?= =?UTF-8?q?=E3=82=A4=E3=83=B3=E3=83=9D=E3=83=BC=E3=83=88=E3=81=A7=E3=81=8D?= =?UTF-8?q?=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=20from=20src=20import=20..?= =?UTF-8?q?.=E3=82=92=E7=9B=B4=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pytest.ini | 3 +++ tests/test_emoji.py | 10 ++++++---- tests/test_misskey.py | 4 +--- tests/test_ngwords.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..81021ad --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +pythonpath = "src" +testpaths = ["tests"] diff --git a/tests/test_emoji.py b/tests/test_emoji.py index 7636060..cfc531b 100644 --- a/tests/test_emoji.py +++ b/tests/test_emoji.py @@ -1,20 +1,22 @@ import pytest - -from src import emojis +import emojis messages = ["response.jsonは{'triggers': [], 'others': []}の形にしてください。", "response.jsonのトリガーのキーはkeywordsとemojiにしてください。", - "[Errno 2] No such file or directory: 'aaaaaaaa'"] + "[Errno 2] No such file or directory: 'aaaaaaaa'", + "Invalid type for path: . Expected str, PathLike, or NoneType." +] @pytest.mark.parametrize("input,msg", (["aaaaaaaa", messages[2]], - [[], messages[0]], + [[], messages[3]], [{"d": "a"}, messages[0]], [{"triggers": [{}], "others": []}, messages[1]])) def test_emojiset_error(input, msg): error_cls = FileNotFoundError if isinstance(input, str) else emojis.ConfigJsonError with pytest.raises(error_cls) as e: emojis.EmojiSet(input) + print(msg) assert str(e.value) == msg diff --git a/tests/test_misskey.py b/tests/test_misskey.py index 46abd3c..af9368c 100644 --- a/tests/test_misskey.py +++ b/tests/test_misskey.py @@ -1,8 +1,6 @@ import json -import sys -sys.path.append("./src") -from src import misskey_api as misskey # NOQA +import misskey_api as misskey # NOQA def test_can_renote(): diff --git a/tests/test_ngwords.py b/tests/test_ngwords.py index 79d43e5..f1f3406 100644 --- a/tests/test_ngwords.py +++ b/tests/test_ngwords.py @@ -1,4 +1,4 @@ -from src import ngwords +import ngwords ng = {"r-18", "荒らす", "twitter", "ワード"} From 140acb91424d0bc8539747bc344679ac6ee477c4 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:26:00 +0900 Subject: [PATCH 39/55] =?UTF-8?q?fix(test):=20=E6=AD=A3=E3=81=97=E3=81=84?= =?UTF-8?q?=E5=88=A4=E5=AE=9A=E3=81=8C=E3=81=A7=E3=81=8D=E3=81=A6=E3=81=84?= =?UTF-8?q?=E3=81=AA=E3=81=84=20=E4=BE=8B=E5=A4=96=E3=81=AA=E3=81=A9?= =?UTF-8?q?=E3=82=92=E5=80=8B=E5=88=A5=E3=81=AB=E6=9B=B8=E3=81=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_emoji.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/test_emoji.py b/tests/test_emoji.py index cfc531b..2e9b1df 100644 --- a/tests/test_emoji.py +++ b/tests/test_emoji.py @@ -1,22 +1,18 @@ import pytest -import emojis +import emojis -messages = ["response.jsonは{'triggers': [], 'others': []}の形にしてください。", - "response.jsonのトリガーのキーはkeywordsとemojiにしてください。", - "[Errno 2] No such file or directory: 'aaaaaaaa'", - "Invalid type for path: . Expected str, PathLike, or NoneType." +inputs = [ + ["aaa", "[Errno 2] No such file or directory: 'aaa'", FileNotFoundError], + [[], "Invalid type for path: . Expected str, PathLike, or NoneType.", TypeError], + [{"a": "b"}, "response.jsonは{'triggers': [], 'others': []}の形にしてください。", emojis.ConfigJsonError], + [{"triggers": [{}], "others": []}, "response.jsonのトリガーのキーはkeywordsとemojiにしてください。", emojis.ConfigJsonError] ] -@pytest.mark.parametrize("input,msg", - (["aaaaaaaa", messages[2]], - [[], messages[3]], - [{"d": "a"}, messages[0]], - [{"triggers": [{}], "others": []}, messages[1]])) -def test_emojiset_error(input, msg): - error_cls = FileNotFoundError if isinstance(input, str) else emojis.ConfigJsonError - with pytest.raises(error_cls) as e: +@pytest.mark.parametrize("input,msg,exception", inputs) +def test_emojiset_error(input, msg, exception): + with pytest.raises(exception) as e: emojis.EmojiSet(input) print(msg) assert str(e.value) == msg From 945b67a412e2a3dd87539efbf946755d2c9947f1 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:28:30 +0900 Subject: [PATCH 40/55] =?UTF-8?q?chore:=20=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/userdb.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/userdb.py b/src/userdb.py index 85fd993..b368665 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -56,8 +56,10 @@ async def _migrate(self) -> None: if __name__ == "__main__": - import asyncio + import asyncio, environs - DB_URL = "redis://localhost:6379" - db = UserDB(DB_URL) - print(type(asyncio.run(db.get_user_by_name("ffdi")))) + db_url = environs.Settings().db_url + if db_url is None: + exit(print("db_type is not redis")) + db = UserDB(str(db_url)) + print(asyncio.run(db.get_user_by_name("ffdi"))) From 704608402c3d25dab8d7d29ff8cfa9d154fb3e38 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:28:58 +0900 Subject: [PATCH 41/55] lint(test): test_ng_words.py --- tests/test_ngwords.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ngwords.py b/tests/test_ngwords.py index f1f3406..1a74672 100644 --- a/tests/test_ngwords.py +++ b/tests/test_ngwords.py @@ -1,6 +1,5 @@ import ngwords - ng = {"r-18", "荒らす", "twitter", "ワード"} allow = {"除外ワード", "除外ワード2"} From 1bbc7d647a85ce7c0a8dd894c2e8dd7ff8ddb3a7 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:46:19 +0900 Subject: [PATCH 42/55] =?UTF-8?q?fix:=20=E8=A6=8B=E3=81=AB=E3=81=8F?= =?UTF-8?q?=E3=81=84=E5=86=85=E5=8C=85=E8=A1=A8=E8=A8=98=E3=82=92=E5=89=8A?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ngwords.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/ngwords.py b/src/ngwords.py index ebbec9f..35ce0fd 100644 --- a/src/ngwords.py +++ b/src/ngwords.py @@ -23,9 +23,17 @@ def __getitem__(self, key) -> set: def _load(self) -> None: data = self.raw.split("\n") - data = [j.lower() for j in data if j != ""] - self._ng = {j for j in data if (j[0] != "-") and (j[0] != "#")} - self._allow = {j[1:] for j in data if j[0] == "-"} + data = [i.lower() for i in data if i != ""] + + ng = set() + allow = set() + for i in data: + if i[0] == "-": + allow.add(i[1:].lstrip(" ")) + elif i[0] != "#": + ng.add(i) + self._ng = ng + self._allow = allow def match(self, text) -> bool: text = text.lower() From 60ca52ae1c4ddf40abb86671f41c1fc5a12689c8 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:46:38 +0900 Subject: [PATCH 43/55] =?UTF-8?q?refactor:=20=E5=A4=89=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=81=AE=E6=94=B9=E5=96=84=20=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_ngwords.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_ngwords.py b/tests/test_ngwords.py index 1a74672..4f2d801 100644 --- a/tests/test_ngwords.py +++ b/tests/test_ngwords.py @@ -5,11 +5,11 @@ def test_ngwords(): - _ng = ngwords.NGWords("tests/test_ngwords.txt") - assert _ng.all_ng_words == ng - assert _ng.all_excluded_words == allow + words = ngwords.NGWords("tests/test_ngwords.txt") + assert words.all_ng_words == ng + assert words.all_excluded_words == allow for word in ng: - assert _ng.match(word) + assert words.match(word) for word in allow: - assert not _ng.match(word) - assert _ng.why("くぁwせdR-18rftgyふじこ") == "r-18" + assert not words.match(word) + assert words.why("くぁwせdR-18rftgyふじこ") == "r-18" From adcb27bc47d9cec445d22ee19b2712e98038073c Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:48:27 +0900 Subject: [PATCH 44/55] =?UTF-8?q?chore:=20.gitignore=E3=82=92=E6=95=B4?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7e5f432..13d06aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,16 @@ +# venv env/ + +# cache __pycache__ +.pytest_cache +# private *.env -.pytest_cache !sample.env data/users.pickle +ngWords.txt -!.vscode/settings.json +# vscode .vscode/ - -ngWords.txt +!.vscode/settings.json From f03502c5da294595bb213af13079c693c771ee9b Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:30:41 +0900 Subject: [PATCH 45/55] =?UTF-8?q?fix:=20response.json=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A9=E3=83=BC=E3=83=9E=E3=83=83=E3=83=88=E3=81=AE=E5=88=A4?= =?UTF-8?q?=E5=AE=9A=E3=81=8C=E3=81=A7=E3=81=8D=E3=81=A6=E3=81=84=E3=81=AA?= =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/emojis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emojis.py b/src/emojis.py index ed4d2bf..5e2f560 100644 --- a/src/emojis.py +++ b/src/emojis.py @@ -21,7 +21,7 @@ def _check_format(self, json: dict) -> None: "response.jsonは{'triggers': [], 'others': []}の形にしてください。" ) - if any([sorted(i.keys()) != ("emoji", "keywords") for i in json["triggers"]]): + if any([sorted(i.keys()) != ["emoji", "keywords"] for i in json["triggers"]]): raise ConfigJsonError( "response.jsonのトリガーのキーはkeywordsとemojiにしてください。" ) From e9c2fcb2b0dd19b0fb8ce48a85f0218acc80e64e Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:31:09 +0900 Subject: [PATCH 46/55] =?UTF-8?q?fix:=20Counter=E3=81=AE=E3=83=87=E3=82=B3?= =?UTF-8?q?=E3=83=AC=E3=83=BC=E3=82=BF=E3=83=BC=E3=81=8C=E5=A3=8A=E3=82=8C?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index af050ab..e6e2810 100644 --- a/src/utils.py +++ b/src/utils.py @@ -39,7 +39,7 @@ def __init__(self, counter, do) -> None: self.do = do def __call__(self, f) -> Any: - def wrapper(*args: f.args, **kwargs): + def wrapper(*args, **kwargs): self._now += 1 if self._now == self.count: self._now = 0 From 8c3da9904074d83aad7e70f594477bb672291cdd Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:31:30 +0900 Subject: [PATCH 47/55] =?UTF-8?q?feat:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=ABpong=E3=82=B5=E3=83=BC=E3=83=90=E3=83=BC?= =?UTF-8?q?=E3=81=AE=E8=A8=AD=E5=AE=9A=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/environs.py | 9 ++++++++- src/keep_alive.py | 9 ++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/environs.py b/src/environs.py index accf3bb..f22a0ee 100644 --- a/src/environs.py +++ b/src/environs.py @@ -1,7 +1,9 @@ +from ipaddress import IPv4Address from pathlib import Path from typing import Literal, Optional -from pydantic import DirectoryPath, RedisDsn, Field, model_validator +from pydantic import DirectoryPath, Field, model_validator +from pydantic.networks import IPvAnyAddress, RedisDsn from pydantic_settings import BaseSettings, SettingsConfigDict @@ -18,9 +20,14 @@ class Settings(DotenvSettings): config_dir: DirectoryPath = Path("./config") run_server: bool = False + server_host: IPvAnyAddress = IPv4Address("0.0.0.0") # type: ignore + server_port: int = 8000 @model_validator(mode="after") def required_when_redis(self): if self.db_url is None and self.db_type == "redis": raise ValueError("DB_URL is required when DB_TYPE is redis") + + if not (self.server_port > 0 and self.server_port < 65536): + raise ValueError("SERVER_PORT must be between 1 and 65535") return self diff --git a/src/keep_alive.py b/src/keep_alive.py index 86acf1d..92ce11f 100644 --- a/src/keep_alive.py +++ b/src/keep_alive.py @@ -3,10 +3,9 @@ from flask import Flask, jsonify +import environs import logging_styles import mainbot -import environs - app = Flask("app") @@ -18,13 +17,13 @@ def pong(): return jsonify({"message": "Pong!"}) -def run_server(): - app.run(host="0.0.0.0", port=8080) +def run_server(host: str, port: int): + app.run(host=host, port=port) if __name__ == "__main__": config = environs.Settings() if config.run_server: - Process(target=run_server).start() + Process(target=run_server, args=(str(config.server_host), config.server_port)).start() logger.info("Web server started!") asyncio.run(mainbot.Bot(config).start_bot()) From d49d9b47556d4a88844c0d4bd2b3e6f5428faf0e Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:41:50 +0900 Subject: [PATCH 48/55] =?UTF-8?q?feat:=20DB=E3=81=A8=E3=81=AE=E6=8E=A5?= =?UTF-8?q?=E7=B6=9A=E3=82=92=E7=A2=BA=E8=AA=8D=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mainbot.py | 4 ++++ src/userdb.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/mainbot.py b/src/mainbot.py index fc37f95..4e506e2 100644 --- a/src/mainbot.py +++ b/src/mainbot.py @@ -111,6 +111,10 @@ async def start_bot(self): "body": {"channel": "hybridTimeline", "id": "1"}, } + pong = await self.db.ping() + if not pong: + raise Exception("DB connection failed.") + while True: async with websockets.connect(streaming_api, user_agent_header=USER_AGENT) as ws: # self.on_open(ws) diff --git a/src/userdb.py b/src/userdb.py index b368665..6e1eb98 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -26,6 +26,9 @@ def __init__(self, redis_url: str) -> None: self._db_url = redis_url UserInfo.Meta.database = get_redis_connection(url=self._db_url) # type: ignore + async def ping(self): + return await UserInfo.db().ping() + async def get_all_users(self) -> list[UserInfo]: all_pks = await UserInfo.all_pks() return await asyncio.gather(*[UserInfo.get(i) async for i in all_pks]) From ff5738c021f7bd6fee0c579968a5851e551f2e69 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:45:43 +0900 Subject: [PATCH 49/55] =?UTF-8?q?chore:=20=E3=82=B5=E3=83=B3=E3=83=97?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E3=82=92?= =?UTF-8?q?=E6=9C=80=E6=96=B0=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sample.env | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sample.env b/sample.env index 43136b0..5f2dd68 100644 --- a/sample.env +++ b/sample.env @@ -2,6 +2,9 @@ HOST=misskey.example.com SECRET_TOKEN=misskey_token CONFIG_DIR=./config +DB_TYPE=redis +DB_URL=redis://localhost:6379 + RUN_SERVER=false -DB_TYPE=pickle -DB_URL=ssss +SERVER_HOST=0.0.0.0 +SERVER_PORT=8000 From 8f824aee2f3d52e13e5c6490159be91a578e4382 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:52:19 +0900 Subject: [PATCH 50/55] refactor: sample.env -> .env.example --- sample.env => .env.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sample.env => .env.example (100%) diff --git a/sample.env b/.env.example similarity index 100% rename from sample.env rename to .env.example From 05db3677d43e52803a9411e517d4c16fa3680b95 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:53:30 +0900 Subject: [PATCH 51/55] =?UTF-8?q?refactor:=20db=E9=96=A2=E9=80=A3=E3=81=AE?= =?UTF-8?q?=E5=BC=95=E6=95=B0=E3=81=AE=E5=90=8D=E5=89=8D=E3=82=92=E6=94=B9?= =?UTF-8?q?=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/userdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/userdb.py b/src/userdb.py index 6e1eb98..b2706c9 100644 --- a/src/userdb.py +++ b/src/userdb.py @@ -33,15 +33,15 @@ async def get_all_users(self) -> list[UserInfo]: all_pks = await UserInfo.all_pks() return await asyncio.gather(*[UserInfo.get(i) async for i in all_pks]) - async def get_user_by_id(self, user_id: str) -> UserInfo | None: + async def get_user_by_id(self, id: str) -> UserInfo | None: try: - return await UserInfo.get(user_id) + return await UserInfo.get(id) except NotFoundError: return None - async def get_user_by_name(self, aas: str) -> UserInfo | None: + async def get_user_by_name(self, name: str) -> UserInfo | None: await self._migrate() - found = UserInfo.find(UserInfo.user_name == aas) + found = UserInfo.find(UserInfo.user_name == name) try: return await found.first() # type: ignore except NotFoundError: From 73e94dd6d0bb80f294e7a366d3ae4ca1fe0d116d Mon Sep 17 00:00:00 2001 From: fffena <132272963+fffena@users.noreply.github.com> Date: Fri, 14 Jun 2024 06:51:23 +0000 Subject: [PATCH 52/55] =?UTF-8?q?refactor:=20validator=E3=81=AE=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E5=90=8D=E3=82=92=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/environs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/environs.py b/src/environs.py index f22a0ee..84b31e5 100644 --- a/src/environs.py +++ b/src/environs.py @@ -24,7 +24,7 @@ class Settings(DotenvSettings): server_port: int = 8000 @model_validator(mode="after") - def required_when_redis(self): + def validate_environ(self): if self.db_url is None and self.db_type == "redis": raise ValueError("DB_URL is required when DB_TYPE is redis") From f0f837375a29000edfa49cf3c73fcac7dd8e95e5 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:29:50 +0900 Subject: [PATCH 53/55] =?UTF-8?q?chore:=20.gitignore=E3=81=AE.vscode?= =?UTF-8?q?=E3=83=87=E3=82=A3=E3=83=AC=E3=82=AF=E3=83=88=E3=83=AA=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 13d06aa..f4510df 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ data/users.pickle ngWords.txt # vscode -.vscode/ +.vscode/* !.vscode/settings.json From df0c960f8f39e9058c5dc7e96fd199f0a259b585 Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:33:52 +0900 Subject: [PATCH 54/55] =?UTF-8?q?chore:=20gitignore=E3=81=AB*.rdb=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f4510df..f577d01 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,11 @@ __pycache__ # private *.env -!sample.env data/users.pickle ngWords.txt # vscode .vscode/* !.vscode/settings.json + +*.rdb From 1adc93c9425679519b5d3a0bace98b64f64a9cdf Mon Sep 17 00:00:00 2001 From: ujex256 <105550500+ujex256@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:36:05 +0900 Subject: [PATCH 55/55] =?UTF-8?q?fix:=20=E4=BD=BF=E3=81=A3=E3=81=A6?= =?UTF-8?q?=E3=81=84=E3=81=AA=E3=81=84=E3=83=86=E3=82=B9=E3=83=88=E7=94=A8?= =?UTF-8?q?=E3=81=AE=E3=82=B3=E3=83=BC=E3=83=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ngwords.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ngwords.py b/src/ngwords.py index 35ce0fd..7fd4647 100644 --- a/src/ngwords.py +++ b/src/ngwords.py @@ -54,8 +54,3 @@ def all_ng_words(self) -> set: @property def all_excluded_words(self) -> set: return self._allow - - -if __name__ == "__main__": - print(NGWords(r"ng_words/ngWords.txt").match("r-18")) - print(NGWords(r"ngWords_Hiraassssssss.txt").all_ng_words) # FileNotFound