From 1b535fb6e93dd8bba56ed6cf2f12d6146cea044a Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Sun, 1 Jun 2025 18:18:25 +0800 Subject: [PATCH 1/9] move all synchronization tools into one single registry --- src/bigmeow/discord.py | 312 +++++++++++++++++++++++++-------------- src/bigmeow/main.py | 48 +++--- src/bigmeow/meow.py | 59 +++++--- src/bigmeow/scheduler.py | 21 +-- src/bigmeow/settings.py | 47 +++--- src/bigmeow/telegram.py | 117 ++++++++++----- src/bigmeow/web.py | 31 +++- 7 files changed, 414 insertions(+), 221 deletions(-) diff --git a/src/bigmeow/discord.py b/src/bigmeow/discord.py index ad7f010..e61b2d8 100644 --- a/src/bigmeow/discord.py +++ b/src/bigmeow/discord.py @@ -2,11 +2,14 @@ import json import queue from contextlib import suppress +from functools import partial from io import StringIO from multiprocessing.synchronize import Event as Event +from typing import Any import discord import httpx +from discord.ext import commands from structlog.stdlib import BoundLogger import bigmeow.settings as settings @@ -27,167 +30,225 @@ from bigmeow.settings import MeowCommand -def client_init() -> discord.Client: - intents = discord.Intents.default() - intents.messages = True - intents.message_content = True - intents.members = True - return discord.Client(intents=discord.Intents(messages=True, message_content=True)) +async def on_ready( + bot: commands.Bot, messages: queue.Queue, logger: BoundLogger +) -> None: + logger.info("DISCORD: Ready for requests") + if settings.DEBUG: + user = await bot.fetch_user(settings.DISCORD_USER) + + logger.info("DISCORD: Sending up message to owner", user=settings.DISCORD_USER) + if bot.user: + asyncio.create_task( + user.send(f"Bot {bot.user.mention} is up\n{meow_say('Hello~')}") + ) -client = client_init() + asyncio.create_task(coroutine_repeat_queue(messages_consume, bot, messages, logger)) async def run( - exit_event: Event, - client: discord.Client = client, + sync_store: settings.SyncStore, logger: BoundLogger = get_logger(__name__), ) -> None: logger.info("DISCORD: Starting") - async with client: - asyncio.create_task(client.start(settings.DISCORD_TOKEN)) - await asyncio.to_thread(exit_event.wait) + bot = commands.Bot( + "!", + intents=discord.Intents(messages=True, message_content=True), + ) + bot.add_listener( + partial(on_ready, bot=bot, messages=sync_store.discord.messages, logger=logger), + "on_ready", + ) + bot.add_listener( + partial( + on_message, + bot=bot, + cats=sync_store.cats, + lock=sync_store.cat_lock, + logger=logger, + ), + "on_message", + ) + bot.add_command( + command_decorator(petrol_fetch, MeowCommand.PETROL, sync_store, logger) + ) + bot.add_command(command_decorator(say_create, MeowCommand.SAY, sync_store, logger)) + bot.add_command( + command_decorator(think_create, MeowCommand.THINK, sync_store, logger) + ) + bot.add_command( + command_decorator(prompt_create, MeowCommand.PROMPT, sync_store, logger) + ) + bot.add_command( + command_decorator(blockedornot_fetch, MeowCommand.ISBLOCKED, sync_store, logger) + ) + bot.add_command(command_decorator(fact_fetch, MeowCommand.FACT, sync_store, logger)) + bot.add_command( + command_decorator(remind_submit, MeowCommand.REMIND, sync_store, logger) + ) + + async with bot: + asyncio.create_task(bot.start(settings.DISCORD_TOKEN)) + + await asyncio.to_thread(sync_store.exit_event.wait) logger.info("DISCORD: Stopping") - await client.close() + await bot.close() -async def messages_consume(client: discord.Client, logger: BoundLogger) -> None: - with suppress(queue.Empty): +async def messages_consume( + bot: commands.Bot, messages: queue.Queue, logger: BoundLogger +) -> None: + with suppress(queue.Empty), suppress(AssertionError): data = await asyncio.to_thread( - settings.discord_messages.get, timeout=settings.QUEUE_TIMEOUT + # FIXME figure something out + messages.get, + timeout=settings.QUEUE_TIMEOUT, ) logger.info("DISCORD: Processing messages from queue", data=data) try: - channel = await client.fetch_channel(data["channel_id"]) + channel = await bot.fetch_channel(data["channel_id"]) except Exception as e: logger.error("DISCORD: Invalid channel", data=data) logger.exception(e) # type: ignore try: message = await channel.fetch_message(data["message_id"]) # type: ignore - except Exception: - logger.info("DISCORD: Unable to find message to reply to", data=data) + except Exception as e: + logger.error("DISCORD: Unable to find message to reply to", data=data) + logger.exception(e) # type: ignore message = None - asyncio.create_task(text_send(data["content"], reference=message)) # type: ignore + asyncio.create_task(text_send(data["content"], channel, message)) -@client.event -async def on_message( - message: discord.Message, - client: discord.Client = client, - logger: BoundLogger = get_logger(__name__), -) -> None: - if message.author == client.user: - return +def command_decorator( + func, command: MeowCommand, sync_store: settings.SyncStore, logger: BoundLogger +): + @commands.command(command.value) + async def inner(*args, **kwargs): + return await func(*args, **kwargs, sync_store=sync_store, logger=logger) - logger.info("DISCORD: Received a message", message=message) + return inner - async with httpx.AsyncClient() as aclient: - if message_contains(message.content, str(MeowCommand.PETROL)): - asyncio.create_task( - text_send(await meow_petrol(aclient), reference=message) - ) - elif message_contains(message.content, str(MeowCommand.SAY)): - asyncio.create_task( - text_send( - meow_say(message.content.replace(str(MeowCommand.SAY), "").strip()), - reference=message, - ) - ) +async def petrol_fetch( + context: commands.Context, sync_store: settings.SyncStore, logger: BoundLogger +) -> None: + logger.info("DISCORD: Received a command", message=context.message) - elif message_contains(message.content, str(MeowCommand.PROMPT)): - await meow_prompt( - aclient, - message.content.replace(str(MeowCommand.PROMPT), "").strip(), - channel="discord", - destination=json.dumps((message.channel.id, message.id)), + async with httpx.AsyncClient() as client: + asyncio.create_task( + text_send( + await meow_petrol(client, sync_store.petrol, sync_store.petrol_lock), + context.message.channel, + context.message, ) + ) - elif message_contains(message.content, str(MeowCommand.THINK)): - asyncio.create_task( - text_send( - meow_say( - message.content.replace(str(MeowCommand.THINK), "").strip(), - is_cowthink=True, - ), - reference=message, - ) - ) +async def say_create( + context: commands.Context, + *args: str, + sync_store: settings.SyncStore, + logger: BoundLogger, +) -> None: + asyncio.create_task( + text_send( + meow_say(" ".join(args).strip()), context.message.channel, context.message + ) + ) - elif message_contains(message.content, str(MeowCommand.FACT)): - asyncio.create_task(text_send(await meow_fact(aclient), reference=message)) - elif message_contains(message.content, str(MeowCommand.ISBLOCKED)): - asyncio.create_task( - text_send( - await meow_blockedornot( - aclient, - message.content.replace(str(MeowCommand.ISBLOCKED), "").strip(), - ), - reference=message, - ) - ) +async def prompt_create( + context: commands.Context, + *args: str, + sync_store: settings.SyncStore, + logger: BoundLogger, +) -> None: + async with httpx.AsyncClient() as client: + await meow_prompt( + client, + " ".join(args).strip(), + channel="discord", + destination=json.dumps((context.message.channel.id, context.message.id)), + ) - elif message_contains(message.content, str(MeowCommand.REMIND)): - asyncio.create_task(process_remind(message, client, logger)) - elif message_contains(message.content, "meow", is_command=False): - logger.info("DISCORD: Sending a cat photo", message=message) - asyncio.create_task( - message.channel.send( - "photo from https://cataas.com/", - file=discord.File( - await meow_fetch_photo(aclient), - description="photo from https://cataas.com/", - filename="meow.png", - ), - reference=message, - ) - ) +async def think_create( + context: commands.Context, + *args: str, + sync_store: settings.SyncStore, + logger: BoundLogger, +) -> None: + asyncio.create_task( + text_send( + meow_say(" ".join(args).strip(), is_cowthink=True), + context.message.channel, + context.message, + ) + ) -@client.event -async def on_ready( - client: discord.Client = client, logger: BoundLogger = get_logger(__name__) +async def blockedornot_fetch( + context: commands.Context, + url: str, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: - logger.info("DISCORD: Ready for requests") + async with httpx.AsyncClient() as client: + asyncio.create_task( + text_send( + await meow_blockedornot(client, url), + context.message.channel, + context.message, + ) + ) - if settings.DEBUG: - user = await client.fetch_user(settings.DISCORD_USER) - logger.info("DISCORD: Sending up message to owner", user=settings.DISCORD_USER) - if client.user: - asyncio.create_task( - user.send(f"Bot {client.user.mention} is up\n{meow_say('Hello~')}") +async def fact_fetch( + context: commands.Context, sync_store: settings.SyncStore, logger: BoundLogger +) -> None: + async with httpx.AsyncClient() as client: + asyncio.create_task( + text_send( + await meow_fact( + client, + sync_store.facts, + sync_store.fact_lock, + ), + context.message.channel, + context.message, ) - - asyncio.create_task(coroutine_repeat_queue(messages_consume, client, logger)) + ) -async def process_remind( - message: discord.Message, client: discord.Client, logger: BoundLogger +async def remind_submit( + context: commands.Context, + *args: str, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: - logger.info("DISCORD: Processing remind request", message=message) + logger.info("DISCORD: Processing remind request", message=context.message) try: asyncio.create_task( text_send( await meow_remind( - message.content.replace(str(MeowCommand.REMIND), "").strip(), - settings.discord_messages, + " ".join(args).strip(), + sync_store.tasks, + sync_store.discord.messages, lambda content: { "content": content, - "channel_id": message.channel.id, - "message_id": message.id, + "channel_id": context.message.channel.id, + "message_id": context.message.id, }, ), - reference=message, + context.message.channel, + context.message, ) ) @@ -195,15 +256,48 @@ async def process_remind( asyncio.create_task( text_send( "Fail to schedule message, please check format again", - reference=message, + context.message.channel, + context.message, ) ) -async def text_send(content: str, reference: discord.Message) -> None: +async def on_message( + message: discord.Message, + bot: commands.Bot, + cats: settings.CatCache, + lock: settings.Lock, + logger: BoundLogger, +) -> None: + if message.author == bot.user: + return + elif (ctx := await bot.get_context(message)) and ctx.valid: + return + + logger.info("DISCORD: Received a message", message=message) + + async with httpx.AsyncClient() as client: + if message_contains(message.content, "meow", is_command=False): + logger.info("DISCORD: Sending a cat photo", message=message) + asyncio.create_task( + message.channel.send( + "photo from https://cataas.com/", + file=discord.File( + await meow_fetch_photo(client, cats, lock), + description="photo from https://cataas.com/", + filename="meow.png", + ), + reference=message, + ) + ) + + +async def text_send( + content: str, channel: Any, reference: discord.Message | None +) -> None: asyncio.create_task( - reference.channel.send( - reference=reference, + channel.send( + reference=reference, # type: ignore **( { "file": discord.File( @@ -213,6 +307,6 @@ async def text_send(content: str, reference: discord.Message) -> None: } if len(content) > 2000 else {"content": content} - ), # type: ignore + ), ) ) \ No newline at end of file diff --git a/src/bigmeow/main.py b/src/bigmeow/main.py index a4cd2d0..ff22c5c 100644 --- a/src/bigmeow/main.py +++ b/src/bigmeow/main.py @@ -1,9 +1,10 @@ import asyncio +import multiprocessing import signal -import threading from collections.abc import Callable from concurrent.futures import Future, ProcessPoolExecutor from dataclasses import dataclass +from datetime import date from types import FrameType from typing import Annotated, Any @@ -16,18 +17,17 @@ @dataclass class ShutdownHandler: - exit_event: threading.Event + sync_store: settings.SyncStore logger: BoundLogger def __call__(self, signum: int | None, frame: FrameType | None) -> None: self.logger.info("MAIN: Sending exit event to all tasks in pool") - self.exit_event.set() + self.sync_store.exit_event.set() @dataclass class DoneHandler: name: str - exit_event: threading.Event logger: BoundLogger shutdown_handler: ShutdownHandler @@ -44,14 +44,14 @@ def __call__(self, future: Future) -> None: self.shutdown_handler(None, None) -def process_run(func, exit_event: threading.Event, *arguments) -> None: - asyncio.run(func(exit_event, *arguments)) +def process_run(func, sync_store: settings.SyncStore, *arguments) -> None: + asyncio.run(func(sync_store, *arguments)) def task_submit( run: bool, executor: ProcessPoolExecutor, - exit_event: threading.Event, + sync_store: settings.SyncStore, name: str, func: Callable[..., Any], shutdown_handler: ShutdownHandler, @@ -59,11 +59,9 @@ def task_submit( *arguments: Any, ) -> Future | None: if run: - future = executor.submit(process_run, func, exit_event, *arguments) + future = executor.submit(process_run, func, sync_store, *arguments) - future.add_done_callback( - DoneHandler(name, exit_event, logger, shutdown_handler) - ) + future.add_done_callback(DoneHandler(name, logger, shutdown_handler)) logger.info("MAIN: Task is submitted", name=name, future=future) return future @@ -74,10 +72,26 @@ def main( run_telegram: Annotated[bool, typer.Option(" /--notg")] = True, ) -> None: logger = get_logger(__name__) - exit_event = settings.manager.Event() + + manager = multiprocessing.Manager() + sync_store = settings.SyncStore( + manager.Event(), + settings.TelegramSyncStore(manager.Queue(), manager.Queue()), + settings.DiscordSyncStore(manager.Queue()), + settings.CatCache(), + settings.Lock(manager.Lock()), + settings.FactCache(), + settings.Lock(manager.Lock()), + settings.PetrolPrice( + settings.PetrolLevel(date.min, 0, 0, 0), + settings.PetrolChange(date.min, 0, 0, 0), + ), + settings.Lock(manager.Lock()), + manager.Queue(), + ) with ProcessPoolExecutor(max_workers=10) as executor: - shutdown_handler = ShutdownHandler(exit_event, logger) + shutdown_handler = ShutdownHandler(sync_store, logger) for s in (signal.SIGHUP, signal.SIGTERM, signal.SIGINT): signal.signal(s, shutdown_handler) @@ -86,7 +100,7 @@ def main( bar = task_submit( run_telegram, executor, - exit_event, + sync_store, "bot.telegram", telegram.run, shutdown_handler, @@ -97,7 +111,7 @@ def main( bar = task_submit( run_discord, executor, - exit_event, + sync_store, "bot.discord", discord.run, shutdown_handler, @@ -108,7 +122,7 @@ def main( bar = task_submit( True, executor, - exit_event, + sync_store, "scheduler", scheduler.run, shutdown_handler, @@ -119,7 +133,7 @@ def main( bar = task_submit( True, executor, - exit_event, + sync_store, "web", web.run, shutdown_handler, diff --git a/src/bigmeow/meow.py b/src/bigmeow/meow.py index 6043dab..cfc9f3f 100644 --- a/src/bigmeow/meow.py +++ b/src/bigmeow/meow.py @@ -15,7 +15,7 @@ from bigmeow import settings from bigmeow.common import get_logger -from bigmeow.settings import Latest, PetrolChange, PetrolLevel +from bigmeow.settings import PetrolChange, PetrolLevel, PetrolPrice def meow_sayify(func: Callable) -> Callable: @@ -50,8 +50,8 @@ async def meow_blockedornot( def meowpetrol_update_latest( - current: Latest, incoming: PetrolLevel | PetrolChange -) -> Latest: + current: PetrolPrice, incoming: PetrolLevel | PetrolChange +) -> PetrolPrice: field = None if isinstance(incoming, PetrolLevel): @@ -66,7 +66,10 @@ def meowpetrol_update_latest( @meow_sayify async def meow_fact( - client: httpx.AsyncClient, logger: BoundLogger = get_logger(__name__) + client: httpx.AsyncClient, + facts: settings.FactCache, + lock: settings.Lock, + logger: BoundLogger = get_logger(__name__), ) -> str: url = "https://meowfacts.herokuapp.com/" @@ -74,27 +77,30 @@ async def meow_fact( response = await client.get(url) response_data = response.json() - async with settings.fact_lock: + async with lock: return ( - settings.fact_cache.cache( + facts.cache( f"{response_data.get('data')[0]}\n - https://github.com/wh-iterabb-it/meowfacts" ) if response.status_code == 200 - else settings.fact_cache.get() + else facts.get() ) @meow_sayify async def meow_petrol( - client: httpx.AsyncClient, logger: BoundLogger = get_logger(__name__) + client: httpx.AsyncClient, + petrol: settings.PetrolPrice, + lock: settings.Lock, + logger: BoundLogger = get_logger(__name__), ) -> str: url = "https://storage.data.gov.my/commodities/fuelprice.csv" - async with settings.latest_lock: - if (settings.latest_cache.level.date + timedelta(days=6)) < date.today(): + async with lock: + if (petrol.level.date + timedelta(days=6)) < date.today(): logger.info("MEOW: Fetching the fuel price list", url=url) response = await client.get(url) - settings.latest_cache = reduce( + petrol = reduce( meowpetrol_update_latest, [ PetrolLevel( @@ -112,22 +118,22 @@ async def meow_petrol( ) for row in csv.DictReader(StringIO(response.text)) ], - settings.latest_cache, + petrol, ) return "\n\n".join( ( f"Data sourced from {url}", - f"From {settings.latest_cache.level.date.strftime(settings.DATE_FORMAT)} to " - f"{(settings.latest_cache.level.date + timedelta(days=6)).strftime(settings.DATE_FORMAT)}", + f"From {petrol.level.date.strftime(settings.DATE_FORMAT)} to " + f"{(petrol.level.date + timedelta(days=6)).strftime(settings.DATE_FORMAT)}", ) + tuple( "Price of {} is RM {} per litre ({} from last week)".format( {"ron95": "RON 95", "ron97": "RON 97", "diesel": "diesel"}.get( field ), - getattr(settings.latest_cache.level, field), - "{:+0.2f}".format(getattr(settings.latest_cache.change, field)), + getattr(petrol.level, field), + "{:+0.2f}".format(getattr(petrol.change, field)), ) for field in ("ron95", "ron97", "diesel") ) @@ -135,18 +141,21 @@ async def meow_petrol( async def meow_fetch_photo( - client: httpx.AsyncClient, logger: BoundLogger = get_logger(__name__) + client: httpx.AsyncClient, + cats: settings.CatCache, + lock: settings.Lock, + logger: BoundLogger = get_logger(__name__), ) -> BytesIO: url = "https://cataas.com/cat/says/meow?type=square" logger.info("MEOW: Fetching a cat photo", url=url) response = await client.get(url) - async with settings.cat_lock: + async with lock: return ( - settings.cat_cache.cache(BytesIO(response.read())) + cats.cache(BytesIO(response.read())) if response.status_code == 200 - else settings.cat_cache.get() + else cats.get() ) @@ -157,7 +166,6 @@ async def meow_prompt( destination: str, logger: BoundLogger = get_logger(__name__), ) -> None: - print(client) url = f"https://maker.ifttt.com/trigger/prompt/with/key/{settings.IFTTT_KEY}" data = {"value1": message, "value2": channel, "value3": destination} @@ -167,7 +175,10 @@ async def meow_prompt( async def meow_remind( - message: str, queue: Queue, data_builder: Callable[[str], dict[str, Any]] + message: str, + task_queue: Queue, + message_queue: Queue, + data_builder: Callable[[str], dict[str, Any]], ) -> str: text, when = message.rsplit("@", maxsplit=1) when = dateparser.parse(when, settings={"TIMEZONE": settings.TIMEZONE.zone}) # type: ignore @@ -175,12 +186,12 @@ async def meow_remind( assert when await asyncio.to_thread( - settings.task_queue.put, + task_queue.put, { "func": "bigmeow.scheduler:execute_sync", "trigger": DateTrigger(when, settings.TIMEZONE), "args": ( - queue.put, + message_queue.put, data_builder(meow_say(text)), ), "misfire_grace_time": None, diff --git a/src/bigmeow/scheduler.py b/src/bigmeow/scheduler.py index 6fc5293..1e5a2f8 100644 --- a/src/bigmeow/scheduler.py +++ b/src/bigmeow/scheduler.py @@ -1,8 +1,7 @@ import asyncio from collections.abc import Callable from contextlib import suppress -from queue import Empty -from threading import Event +from queue import Empty, Queue from typing import Any from apscheduler.executors.pool import ProcessPoolExecutor @@ -18,17 +17,19 @@ def execute_sync(callable: Callable[..., Any], *args) -> None: callable(*args) -async def task_consume(scheduler: AsyncIOScheduler, logger: BoundLogger) -> None: +async def task_consume( + scheduler: AsyncIOScheduler, queue: Queue, logger: BoundLogger +) -> None: with suppress(Empty): - task = await asyncio.to_thread( - settings.task_queue.get, timeout=settings.QUEUE_TIMEOUT - ) + task = await asyncio.to_thread(queue.get, timeout=settings.QUEUE_TIMEOUT) logger.info("Retrieved task", **task) scheduler.add_job(**task) -async def run(exit_event: Event, logger: BoundLogger = get_logger(__name__)) -> None: +async def run( + sync_store: settings.SyncStore, logger: BoundLogger = get_logger(__name__) +) -> None: logger.info("SCHEDULER: Starting") scheduler = AsyncIOScheduler( timezone=settings.TIMEZONE, @@ -41,9 +42,11 @@ async def run(exit_event: Event, logger: BoundLogger = get_logger(__name__)) -> scheduler.start() logger.info("SCHEDULER: Ready for requests") - asyncio.create_task(coroutine_repeat_queue(task_consume, scheduler, logger)) + asyncio.create_task( + coroutine_repeat_queue(task_consume, scheduler, sync_store.tasks, logger) + ) - await asyncio.to_thread(exit_event.wait) + await asyncio.to_thread(sync_store.exit_event.wait) logger.info("SCHEDULER: Stopping") diff --git a/src/bigmeow/settings.py b/src/bigmeow/settings.py index 077a4d4..ed90729 100644 --- a/src/bigmeow/settings.py +++ b/src/bigmeow/settings.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import multiprocessing import threading from abc import ABC from ast import literal_eval @@ -8,6 +7,7 @@ from enum import Enum from io import BytesIO from os import environ +from queue import Queue from random import choice, randint, shuffle from typing import NamedTuple @@ -22,7 +22,7 @@ # TODO use proper typing and abstrct to abstract class in py3.12 -class Cat_Cache: +class CatCache: cat_list: list[BytesIO] = [] def cache(self, cat: BytesIO) -> BytesIO: @@ -44,7 +44,7 @@ def get(self) -> BytesIO: return choice(self.cat_list) -class Fact_Cache: +class FactCache: fact_list: list[str] = [] def cache(self, fact: str) -> str: @@ -93,7 +93,7 @@ class PetrolChange(PetrolRow): pass -class Latest(NamedTuple): +class PetrolPrice(NamedTuple): level: PetrolLevel change: PetrolChange @@ -119,18 +119,33 @@ def __str__(self) -> str: return f"{COMMAND_PREFIX}{self.value}" -manager = multiprocessing.Manager() -cat_cache = Cat_Cache() -cat_lock = Lock(manager.Lock()) +@dataclass +class TelegramSyncStore: + messages: Queue + updates: Queue -fact_cache = Fact_Cache() -fact_lock = Lock(manager.Lock()) -latest_cache = Latest( - PetrolLevel(date.min, 0, 0, 0), - PetrolChange(date.min, 0, 0, 0), -) -latest_lock = Lock(manager.Lock()) +@dataclass +class DiscordSyncStore: + messages: Queue + + +@dataclass +class SyncStore: + exit_event: threading.Event + telegram: TelegramSyncStore + discord: DiscordSyncStore + + cats: CatCache + cat_lock: Lock + + facts: FactCache + fact_lock: Lock + + petrol: PetrolPrice + petrol_lock: Lock + + tasks: Queue try: DEBUG = literal_eval(environ.get("DEBUG", "False")) @@ -153,16 +168,12 @@ def __str__(self) -> str: TELEGRAM_USER = environ["TELEGRAM_USER"] TELEGRAM_TOKEN = environ["TELEGRAM_TOKEN"] TELEGRAM_WEB_TOKEN = environ["WEB_TELEGRAM_TOKEN"] -telegram_messages = manager.Queue() -telegram_updates = manager.Queue() DISCORD_TOKEN = environ["DISCORD_TOKEN"] DISCORD_USER = int(environ["DISCORD_USER"]) -discord_messages = manager.Queue() ECHO_WEBHOOK = "/webhook/echo" -task_queue = manager.Queue() TASK_DEFAULT_STORE = "default" TASK_DEFAULT_EXECUTOR = "default" diff --git a/src/bigmeow/telegram.py b/src/bigmeow/telegram.py index af25421..a0904ca 100644 --- a/src/bigmeow/telegram.py +++ b/src/bigmeow/telegram.py @@ -4,7 +4,6 @@ from contextlib import suppress from dataclasses import dataclass from functools import partial -from multiprocessing.synchronize import Event from typing import Any, Callable import httpx @@ -76,18 +75,24 @@ async def blockedornot_fetch( def command_handler_pair( - command: MeowCommand, handler: Callable[..., Any], logger: BoundLogger + command: MeowCommand, + handler: Callable[..., Any], + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> tuple[CommandHandler, MessageHandler]: return CommandHandler( - command.value, partial(handler, logger=logger) + command.value, partial(handler, sync_store=sync_store, logger=logger) ), MessageHandler( Filter(str(command), True), - partial(handler, logger=logger), + partial(handler, sync_store=sync_store, logger=logger), ) async def fact_fetch( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + context: ContextTypes.DEFAULT_TYPE, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing fact request", update=update) @@ -97,7 +102,9 @@ async def fact_fetch( context.bot.send_message( chat_id=update.effective_chat.id, parse_mode=ParseMode.MARKDOWN, - text=await meow_fact(client), + text=await meow_fact( + client, sync_store.facts, sync_store.fact_lock + ), reply_to_message_id=update.message.id, allow_sending_without_reply=True, ) @@ -105,7 +112,10 @@ async def fact_fetch( async def meow_create( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + context: ContextTypes.DEFAULT_TYPE, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: if not (update.message and update.effective_chat): return @@ -115,7 +125,9 @@ async def meow_create( asyncio.create_task( context.bot.send_photo( chat_id=update.effective_chat.id, - photo=await meow_fetch_photo(client), + photo=await meow_fetch_photo( + client, sync_store.cats, sync_store.cat_lock + ), caption="photo from https://cataas.com/", reply_to_message_id=update.message.id, allow_sending_without_reply=True, @@ -123,19 +135,21 @@ async def meow_create( ) -async def messages_consume(application: Application, logger: BoundLogger) -> None: +async def messages_consume( + application: Application, messages: queue.Queue, logger: BoundLogger +) -> None: with suppress(queue.Empty): - message = await asyncio.to_thread( - settings.telegram_messages.get, - timeout=settings.QUEUE_TIMEOUT, - ) + message = await asyncio.to_thread(messages.get, timeout=settings.QUEUE_TIMEOUT) logger.info("TELEGRAM: Processing message reply") asyncio.create_task(application.bot.send_message(**message)) async def petrol_fetch( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + context: ContextTypes.DEFAULT_TYPE, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing petrol request", update=update) @@ -145,7 +159,9 @@ async def petrol_fetch( context.bot.send_message( chat_id=update.effective_chat.id, parse_mode=ParseMode.MARKDOWN, - text=await meow_petrol(client), + text=await meow_petrol( + client, sync_store.petrol, sync_store.petrol_lock + ), reply_to_message_id=update.message.id, allow_sending_without_reply=True, ) @@ -153,11 +169,11 @@ async def petrol_fetch( async def run( - exit_event: Event, + sync_store: settings.SyncStore, application: Application = application, logger: BoundLogger = get_logger(__name__), ) -> None: - await setup(application, logger) + await setup(application, sync_store, logger) async with application: logger.info("TELEGRAM: Starting") @@ -176,34 +192,48 @@ async def run( text=meow_say("Bot is up"), ) - asyncio.create_task(coroutine_repeat_queue(updates_consume, application)) asyncio.create_task( - coroutine_repeat_queue(messages_consume, application, logger) + coroutine_repeat_queue( + updates_consume, application, sync_store.telegram.updates, logger + ) + ) + asyncio.create_task( + coroutine_repeat_queue( + messages_consume, application, sync_store.telegram.messages, logger + ) ) - await asyncio.to_thread(exit_event.wait) + await asyncio.to_thread(sync_store.exit_event.wait) logger.info("TELEGRAM: Stopping") await application.stop() -async def setup(application: Application, logger: BoundLogger) -> None: +async def setup( + application: Application, sync_store: settings.SyncStore, logger: BoundLogger +) -> None: logger.info("TELEGRAM: Initializing application") application.add_handlers( list( - command_handler_pair(MeowCommand.PETROL, petrol_fetch, logger) - + command_handler_pair(MeowCommand.SAY, say_create, logger) - + command_handler_pair(MeowCommand.THINK, think_create, logger) - + command_handler_pair(MeowCommand.PROMPT, prompt_create, logger) - + command_handler_pair(MeowCommand.FACT, fact_fetch, logger) - + command_handler_pair(MeowCommand.ISBLOCKED, blockedornot_fetch, logger) - + command_handler_pair(MeowCommand.REMIND, remind_submit, logger) + command_handler_pair(MeowCommand.PETROL, petrol_fetch, sync_store, logger) + + command_handler_pair(MeowCommand.SAY, say_create, sync_store, logger) + + command_handler_pair(MeowCommand.THINK, think_create, sync_store, logger) + + command_handler_pair( + MeowCommand.PROMPT, prompt_create, sync_store, logger + ) + + command_handler_pair(MeowCommand.FACT, fact_fetch, sync_store, logger) + + command_handler_pair( + MeowCommand.ISBLOCKED, blockedornot_fetch, sync_store, logger + ) + + command_handler_pair( + MeowCommand.REMIND, remind_submit, sync_store, logger + ) ) + [ MessageHandler( Filter("meow", False), - partial(meow_create, logger=logger), + partial(meow_create, sync_store=sync_store, logger=logger), ) ] ) @@ -218,7 +248,10 @@ async def setup(application: Application, logger: BoundLogger) -> None: async def prompt_create( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + _context: ContextTypes.DEFAULT_TYPE, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Dispatching prompt request", update=update) @@ -235,7 +268,10 @@ async def prompt_create( async def say_create( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + context: ContextTypes.DEFAULT_TYPE, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing say request", update=update) @@ -256,7 +292,10 @@ async def say_create( async def remind_submit( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + context: ContextTypes.DEFAULT_TYPE, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing remind request", update=update) @@ -271,7 +310,8 @@ async def remind_submit( update.message.text.replace(MeowCommand.REMIND.telegram(), "") .replace(str(MeowCommand.REMIND), "") .strip(), - settings.telegram_messages, + sync_store.tasks, + sync_store.telegram.messages, lambda content: { "text": content, "chat_id": update.effective_chat.id, # type: ignore @@ -298,7 +338,10 @@ async def remind_submit( async def think_create( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + context: ContextTypes.DEFAULT_TYPE, + sync_store: settings.SyncStore, + logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing think request", update=update) @@ -319,13 +362,15 @@ async def think_create( ) -async def updates_consume(application: Application) -> None: +async def updates_consume( + application: Application, updates: queue.Queue, logger: BoundLogger +) -> None: with suppress(queue.Empty): asyncio.create_task( application.update_queue.put( Update.de_json( await asyncio.to_thread( - settings.telegram_updates.get, + updates.get, timeout=settings.QUEUE_TIMEOUT, ), application.bot, diff --git a/src/bigmeow/web.py b/src/bigmeow/web.py index 7f935d9..862ffed 100644 --- a/src/bigmeow/web.py +++ b/src/bigmeow/web.py @@ -1,6 +1,6 @@ import asyncio import json -from multiprocessing.synchronize import Event +from contextlib import asynccontextmanager from typing import Annotated import aiohttp @@ -14,7 +14,18 @@ from bigmeow.common import get_logger from bigmeow.meow import meow_say -app = FastAPI() + +@asynccontextmanager +async def lifespan(app: FastAPI): + if not hasattr(app.state, "sync_store") and isinstance( + app.state, settings.SyncStore + ): + raise RuntimeError("Runtime sync_store object is missing") + + yield + + +app = FastAPI(lifespan=lifespan) class Logger: @@ -52,10 +63,14 @@ def check_login_is_valid(authorization: str | None) -> bool: return result -async def run(exit_event: Event, logger: BoundLogger = get_logger(__name__)) -> None: +async def run( + sync_store: settings.SyncStore, logger: BoundLogger = get_logger(__name__) +) -> None: + app.state.sync_store = sync_store + server = uvicorn.Server( uvicorn.Config( - "bigmeow.web:app", + app, host="0.0.0.0", port=settings.WEBHOOK_PORT, log_level="info", @@ -72,7 +87,7 @@ async def run(exit_event: Event, logger: BoundLogger = get_logger(__name__)) -> else: raise Exception("Website is unreachable") - await asyncio.to_thread(exit_event.wait) + await asyncio.to_thread(sync_store.exit_event.wait) logger.info("WEB: Webserver is stopping") await server.shutdown() @@ -112,7 +127,7 @@ async def telegram_webhook( logger.info("WEBHOOK: Webhook receives a telegram request") asyncio.create_task( asyncio.to_thread( - settings.telegram_updates.put, + request.app.state.sync_store.telegram.updates.put, await request.json(), ) ) @@ -139,7 +154,7 @@ async def chat_post( asyncio.create_task( asyncio.to_thread( - settings.telegram_messages.put, + request.app.state.sync_store.telegram.messages.put, { "text": meow_say(text), "chat_id": chat_id, @@ -154,7 +169,7 @@ async def chat_post( channel_id, message_id = json.loads(x_destination) asyncio.create_task( asyncio.to_thread( - settings.discord_messages.put, + request.app.state.sync_store.discord.messages.put, { "content": meow_say(text), "channel_id": channel_id, From 1fa017cfcfe27b7b2e8215a13d214533d3c1a7ae Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Sun, 1 Jun 2025 19:14:08 +0800 Subject: [PATCH 2/9] remove Lock class and use a function based context manager instead --- src/bigmeow/common.py | 12 ++++++++++++ src/bigmeow/discord.py | 27 ++++++++++----------------- src/bigmeow/main.py | 6 +++--- src/bigmeow/meow.py | 15 ++++++++------- src/bigmeow/settings.py | 19 +++---------------- 5 files changed, 36 insertions(+), 43 deletions(-) diff --git a/src/bigmeow/common.py b/src/bigmeow/common.py index ce663ca..42354ff 100644 --- a/src/bigmeow/common.py +++ b/src/bigmeow/common.py @@ -1,11 +1,23 @@ import asyncio import logging +import threading from collections.abc import Callable +from contextlib import asynccontextmanager from typing import Any, Awaitable import structlog +@asynccontextmanager +async def async_lock(lock: threading.Lock): + await asyncio.to_thread(lock.acquire) + + try: + yield + finally: + lock.release() + + def message_contains(message: str | None, content: str, is_command=True) -> bool: message = message or "" diff --git a/src/bigmeow/discord.py b/src/bigmeow/discord.py index e61b2d8..10d17dc 100644 --- a/src/bigmeow/discord.py +++ b/src/bigmeow/discord.py @@ -1,6 +1,7 @@ import asyncio import json import queue +import threading from contextlib import suppress from functools import partial from io import StringIO @@ -71,23 +72,15 @@ async def run( ), "on_message", ) + bot.add_command(command_make(petrol_fetch, MeowCommand.PETROL, sync_store, logger)) + bot.add_command(command_make(say_create, MeowCommand.SAY, sync_store, logger)) + bot.add_command(command_make(think_create, MeowCommand.THINK, sync_store, logger)) + bot.add_command(command_make(prompt_create, MeowCommand.PROMPT, sync_store, logger)) bot.add_command( - command_decorator(petrol_fetch, MeowCommand.PETROL, sync_store, logger) - ) - bot.add_command(command_decorator(say_create, MeowCommand.SAY, sync_store, logger)) - bot.add_command( - command_decorator(think_create, MeowCommand.THINK, sync_store, logger) - ) - bot.add_command( - command_decorator(prompt_create, MeowCommand.PROMPT, sync_store, logger) - ) - bot.add_command( - command_decorator(blockedornot_fetch, MeowCommand.ISBLOCKED, sync_store, logger) - ) - bot.add_command(command_decorator(fact_fetch, MeowCommand.FACT, sync_store, logger)) - bot.add_command( - command_decorator(remind_submit, MeowCommand.REMIND, sync_store, logger) + command_make(blockedornot_fetch, MeowCommand.ISBLOCKED, sync_store, logger) ) + bot.add_command(command_make(fact_fetch, MeowCommand.FACT, sync_store, logger)) + bot.add_command(command_make(remind_submit, MeowCommand.REMIND, sync_store, logger)) async with bot: asyncio.create_task(bot.start(settings.DISCORD_TOKEN)) @@ -126,7 +119,7 @@ async def messages_consume( asyncio.create_task(text_send(data["content"], channel, message)) -def command_decorator( +def command_make( func, command: MeowCommand, sync_store: settings.SyncStore, logger: BoundLogger ): @commands.command(command.value) @@ -266,7 +259,7 @@ async def on_message( message: discord.Message, bot: commands.Bot, cats: settings.CatCache, - lock: settings.Lock, + lock: threading.Lock, logger: BoundLogger, ) -> None: if message.author == bot.user: diff --git a/src/bigmeow/main.py b/src/bigmeow/main.py index ff22c5c..022f53c 100644 --- a/src/bigmeow/main.py +++ b/src/bigmeow/main.py @@ -79,14 +79,14 @@ def main( settings.TelegramSyncStore(manager.Queue(), manager.Queue()), settings.DiscordSyncStore(manager.Queue()), settings.CatCache(), - settings.Lock(manager.Lock()), + manager.Lock(), settings.FactCache(), - settings.Lock(manager.Lock()), + manager.Lock(), settings.PetrolPrice( settings.PetrolLevel(date.min, 0, 0, 0), settings.PetrolChange(date.min, 0, 0, 0), ), - settings.Lock(manager.Lock()), + manager.Lock(), manager.Queue(), ) diff --git a/src/bigmeow/meow.py b/src/bigmeow/meow.py index cfc9f3f..2102315 100644 --- a/src/bigmeow/meow.py +++ b/src/bigmeow/meow.py @@ -1,5 +1,6 @@ import asyncio import csv +import threading from datetime import date, timedelta from functools import reduce from io import BytesIO, StringIO @@ -14,7 +15,7 @@ from structlog.stdlib import BoundLogger from bigmeow import settings -from bigmeow.common import get_logger +from bigmeow.common import async_lock, get_logger from bigmeow.settings import PetrolChange, PetrolLevel, PetrolPrice @@ -68,7 +69,7 @@ def meowpetrol_update_latest( async def meow_fact( client: httpx.AsyncClient, facts: settings.FactCache, - lock: settings.Lock, + lock: threading.Lock, logger: BoundLogger = get_logger(__name__), ) -> str: url = "https://meowfacts.herokuapp.com/" @@ -77,7 +78,7 @@ async def meow_fact( response = await client.get(url) response_data = response.json() - async with lock: + async with async_lock(lock): return ( facts.cache( f"{response_data.get('data')[0]}\n - https://github.com/wh-iterabb-it/meowfacts" @@ -91,12 +92,12 @@ async def meow_fact( async def meow_petrol( client: httpx.AsyncClient, petrol: settings.PetrolPrice, - lock: settings.Lock, + lock: threading.Lock, logger: BoundLogger = get_logger(__name__), ) -> str: url = "https://storage.data.gov.my/commodities/fuelprice.csv" - async with lock: + async with async_lock(lock): if (petrol.level.date + timedelta(days=6)) < date.today(): logger.info("MEOW: Fetching the fuel price list", url=url) response = await client.get(url) @@ -143,7 +144,7 @@ async def meow_petrol( async def meow_fetch_photo( client: httpx.AsyncClient, cats: settings.CatCache, - lock: settings.Lock, + lock: threading.Lock, logger: BoundLogger = get_logger(__name__), ) -> BytesIO: url = "https://cataas.com/cat/says/meow?type=square" @@ -151,7 +152,7 @@ async def meow_fetch_photo( logger.info("MEOW: Fetching a cat photo", url=url) response = await client.get(url) - async with lock: + async with async_lock(lock): return ( cats.cache(BytesIO(response.read())) if response.status_code == 200 diff --git a/src/bigmeow/settings.py b/src/bigmeow/settings.py index ed90729..8931581 100644 --- a/src/bigmeow/settings.py +++ b/src/bigmeow/settings.py @@ -1,5 +1,3 @@ -import asyncio -import contextlib import threading from abc import ABC from ast import literal_eval @@ -66,17 +64,6 @@ def get(self) -> str: return choice(self.fact_list) -@dataclass -class Lock(contextlib.AbstractAsyncContextManager): - lock: threading.Lock - - async def __aenter__(self) -> None: - await asyncio.to_thread(self.lock.acquire) - - async def __aexit__(self, exc_type, exc, traceback) -> None: - self.lock.release() - - @dataclass class PetrolRow(ABC): date: date @@ -137,13 +124,13 @@ class SyncStore: discord: DiscordSyncStore cats: CatCache - cat_lock: Lock + cat_lock: threading.Lock facts: FactCache - fact_lock: Lock + fact_lock: threading.Lock petrol: PetrolPrice - petrol_lock: Lock + petrol_lock: threading.Lock tasks: Queue From 11ac3033a1f1eea90d68049c5a12e06a2bb26f12 Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Mon, 2 Jun 2025 17:03:34 +0800 Subject: [PATCH 3/9] move classes to common, and revise discord command registration to match tg --- src/bigmeow/common.py | 146 ++++++++++++++++++++++++++++++++++++--- src/bigmeow/discord.py | 118 +++++++++++++++++++------------ src/bigmeow/main.py | 44 ++++++------ src/bigmeow/meow.py | 31 +++++---- src/bigmeow/scheduler.py | 4 +- src/bigmeow/settings.py | 128 ---------------------------------- src/bigmeow/telegram.py | 96 ++++++++++++++----------- src/bigmeow/web.py | 8 +-- 8 files changed, 310 insertions(+), 265 deletions(-) diff --git a/src/bigmeow/common.py b/src/bigmeow/common.py index 42354ff..0a837bb 100644 --- a/src/bigmeow/common.py +++ b/src/bigmeow/common.py @@ -1,11 +1,146 @@ import asyncio import logging import threading +from abc import ABC from collections.abc import Callable from contextlib import asynccontextmanager -from typing import Any, Awaitable +from datetime import date +from enum import Enum +from io import BytesIO +from queue import Queue +from random import choice, randint, shuffle +from typing import Any, Awaitable, NamedTuple import structlog +from attr import dataclass + +from bigmeow import settings + + +def get_logger(module_name: str) -> structlog.stdlib.BoundLogger: + structlog.configure( + wrapper_class=structlog.make_filtering_bound_logger(logging.DEBUG), + ) + return structlog.get_logger().bind(module=module_name) + + +logger = get_logger(__name__) + + +# TODO use proper typing and abstrct to abstract class in py3.12 +class CatCache: + cat_list: list[BytesIO] = [] + + def cache(self, cat: BytesIO) -> BytesIO: + logger.info("CAT_CACHE: Storing a new photo to cache") + + if len(self.cat_list) > settings.CACHE_LIMIT: + self.cat_list[randint(0, settings.CACHE_LIMIT - 1)] = cat + else: + self.cat_list.append(cat) + + shuffle(self.cat_list) + + return cat + + def get(self) -> BytesIO: + assert len(self.cat_list) > 0 + + logger.info("CAT_CACHE: Retrieve a photo") + return choice(self.cat_list) + + +class FactCache: + fact_list: list[str] = [] + + def cache(self, fact: str) -> str: + logger.info("FACT_CACHE: Storing a new fact to cache") + + if len(self.fact_list) > settings.CACHE_LIMIT: + self.fact_list[randint(0, settings.CACHE_LIMIT - 1)] = fact + else: + self.fact_list.append(fact) + + shuffle(self.fact_list) + + return fact + + def get(self) -> str: + assert len(self.fact_list) > 0 + + logger.info("FACT_CACHE: Retrieve a fact") + return choice(self.fact_list) + + +@dataclass +class PetrolRow(ABC): + date: date + ron95: float + ron97: float + diesel: float + + +class PetrolLevel(PetrolRow): + pass + + +class PetrolChange(PetrolRow): + pass + + +class PetrolPrice(NamedTuple): + level: PetrolLevel + change: PetrolChange + + +class MeowCommand(Enum): + SAY = "meowsay" + PETROL = "meowpetrol" + FACT = "meowfact" + ISBLOCKED = "meowisblocked" + THINK = "meowthink" + PROMPT = "meowprompt" + HELP = "meowhelp" + REMIND = "meowremind" + + def telegram(self) -> str: + COMMAND_PREFIX = "/" + + return f"{COMMAND_PREFIX}{self.value}" + + def __str__(self) -> str: + COMMAND_PREFIX = "!" + + return f"{COMMAND_PREFIX}{self.value}" + + +@dataclass +class TelegramSyncStore: + messages: Queue + updates: Queue + + +@dataclass +class DiscordSyncStore: + messages: Queue + + +@dataclass +class SyncStore: + exit_event: threading.Event + telegram: TelegramSyncStore + discord: DiscordSyncStore + + cats: CatCache + cat_lock: threading.Lock + + facts: FactCache + fact_lock: threading.Lock + + petrol: PetrolPrice + petrol_lock: threading.Lock + + tasks: Queue @asynccontextmanager @@ -30,11 +165,4 @@ async def coroutine_repeat_queue( while True: await coro_func(*args) except asyncio.CancelledError: - pass - - -def get_logger(module_name: str) -> structlog.stdlib.BoundLogger: - structlog.configure( - wrapper_class=structlog.make_filtering_bound_logger(logging.DEBUG), - ) - return structlog.get_logger().bind(module=module_name) \ No newline at end of file + pass \ No newline at end of file diff --git a/src/bigmeow/discord.py b/src/bigmeow/discord.py index 10d17dc..7df373d 100644 --- a/src/bigmeow/discord.py +++ b/src/bigmeow/discord.py @@ -13,8 +13,9 @@ from discord.ext import commands from structlog.stdlib import BoundLogger -import bigmeow.settings as settings +from bigmeow import common, settings from bigmeow.common import ( + MeowCommand, coroutine_repeat_queue, get_logger, message_contains, @@ -28,14 +29,11 @@ meow_remind, meow_say, ) -from bigmeow.settings import MeowCommand async def on_ready( bot: commands.Bot, messages: queue.Queue, logger: BoundLogger ) -> None: - logger.info("DISCORD: Ready for requests") - if settings.DEBUG: user = await bot.fetch_user(settings.DISCORD_USER) @@ -45,19 +43,32 @@ async def on_ready( user.send(f"Bot {bot.user.mention} is up\n{meow_say('Hello~')}") ) + logger.info("DISCORD: Ready for requests") asyncio.create_task(coroutine_repeat_queue(messages_consume, bot, messages, logger)) async def run( - sync_store: settings.SyncStore, + sync_store: common.SyncStore, logger: BoundLogger = get_logger(__name__), ) -> None: - logger.info("DISCORD: Starting") - bot = commands.Bot( "!", intents=discord.Intents(messages=True, message_content=True), ) + + setup(bot, sync_store, logger) + + async with bot: + logger.info("DISCORD: Starting") + asyncio.create_task(bot.start(settings.DISCORD_TOKEN)) + + await asyncio.to_thread(sync_store.exit_event.wait) + + logger.info("DISCORD: Stopping") + await bot.close() + + +def setup(bot: commands.Bot, sync_store: common.SyncStore, logger: BoundLogger) -> None: bot.add_listener( partial(on_ready, bot=bot, messages=sync_store.discord.messages, logger=logger), "on_ready", @@ -72,23 +83,37 @@ async def run( ), "on_message", ) - bot.add_command(command_make(petrol_fetch, MeowCommand.PETROL, sync_store, logger)) - bot.add_command(command_make(say_create, MeowCommand.SAY, sync_store, logger)) - bot.add_command(command_make(think_create, MeowCommand.THINK, sync_store, logger)) - bot.add_command(command_make(prompt_create, MeowCommand.PROMPT, sync_store, logger)) bot.add_command( - command_make(blockedornot_fetch, MeowCommand.ISBLOCKED, sync_store, logger) + command_make( + MeowCommand.PETROL, + petrol_fetch, + logger, + petrol=sync_store.petrol, + lock=sync_store.petrol_lock, + ) + ) + bot.add_command(command_make(MeowCommand.SAY, say_create, logger)) + bot.add_command(command_make(MeowCommand.THINK, think_create, logger)) + bot.add_command(command_make(MeowCommand.PROMPT, prompt_create, logger)) + bot.add_command(command_make(MeowCommand.ISBLOCKED, blockedornot_fetch, logger)) + bot.add_command( + command_make( + MeowCommand.FACT, + fact_fetch, + logger, + facts=sync_store.facts, + lock=sync_store.fact_lock, + ) + ) + bot.add_command( + command_make( + MeowCommand.REMIND, + remind_submit, + logger, + tasks=sync_store.tasks, + messages=sync_store.discord.messages, + ) ) - bot.add_command(command_make(fact_fetch, MeowCommand.FACT, sync_store, logger)) - bot.add_command(command_make(remind_submit, MeowCommand.REMIND, sync_store, logger)) - - async with bot: - asyncio.create_task(bot.start(settings.DISCORD_TOKEN)) - - await asyncio.to_thread(sync_store.exit_event.wait) - - logger.info("DISCORD: Stopping") - await bot.close() async def messages_consume( @@ -119,25 +144,26 @@ async def messages_consume( asyncio.create_task(text_send(data["content"], channel, message)) -def command_make( - func, command: MeowCommand, sync_store: settings.SyncStore, logger: BoundLogger -): +def command_make(command: MeowCommand, func, logger: BoundLogger, **kwargs: Any): @commands.command(command.value) - async def inner(*args, **kwargs): - return await func(*args, **kwargs, sync_store=sync_store, logger=logger) + async def inner(*args_inner, **kwargs_inner): + return await func(*args_inner, **kwargs_inner, **kwargs, logger=logger) return inner async def petrol_fetch( - context: commands.Context, sync_store: settings.SyncStore, logger: BoundLogger + context: commands.Context, + petrol: common.PetrolPrice, + lock: threading.Lock, + logger: BoundLogger, ) -> None: logger.info("DISCORD: Received a command", message=context.message) async with httpx.AsyncClient() as client: asyncio.create_task( text_send( - await meow_petrol(client, sync_store.petrol, sync_store.petrol_lock), + await meow_petrol(client, petrol, lock, logger), context.message.channel, context.message, ) @@ -146,12 +172,13 @@ async def petrol_fetch( async def say_create( context: commands.Context, *args: str, - sync_store: settings.SyncStore, logger: BoundLogger, ) -> None: asyncio.create_task( text_send( - meow_say(" ".join(args).strip()), context.message.channel, context.message + meow_say(" ".join(args).strip()), + context.message.channel, + context.message, ) ) @@ -159,7 +186,6 @@ async def say_create( async def prompt_create( context: commands.Context, *args: str, - sync_store: settings.SyncStore, logger: BoundLogger, ) -> None: async with httpx.AsyncClient() as client: @@ -168,13 +194,13 @@ async def prompt_create( " ".join(args).strip(), channel="discord", destination=json.dumps((context.message.channel.id, context.message.id)), + logger=logger, ) async def think_create( context: commands.Context, *args: str, - sync_store: settings.SyncStore, logger: BoundLogger, ) -> None: asyncio.create_task( @@ -189,13 +215,12 @@ async def think_create( async def blockedornot_fetch( context: commands.Context, url: str, - sync_store: settings.SyncStore, logger: BoundLogger, ) -> None: async with httpx.AsyncClient() as client: asyncio.create_task( text_send( - await meow_blockedornot(client, url), + await meow_blockedornot(client, url, logger), context.message.channel, context.message, ) @@ -203,16 +228,15 @@ async def blockedornot_fetch( async def fact_fetch( - context: commands.Context, sync_store: settings.SyncStore, logger: BoundLogger + context: commands.Context, + facts: common.FactCache, + lock: threading.Lock, + logger: BoundLogger, ) -> None: async with httpx.AsyncClient() as client: asyncio.create_task( text_send( - await meow_fact( - client, - sync_store.facts, - sync_store.fact_lock, - ), + await meow_fact(client, facts, lock, logger), context.message.channel, context.message, ) @@ -222,7 +246,8 @@ async def fact_fetch( async def remind_submit( context: commands.Context, *args: str, - sync_store: settings.SyncStore, + tasks: queue.Queue, + messages: queue.Queue, logger: BoundLogger, ) -> None: logger.info("DISCORD: Processing remind request", message=context.message) @@ -232,13 +257,14 @@ async def remind_submit( text_send( await meow_remind( " ".join(args).strip(), - sync_store.tasks, - sync_store.discord.messages, + tasks, + messages, lambda content: { "content": content, "channel_id": context.message.channel.id, "message_id": context.message.id, }, + logger, ), context.message.channel, context.message, @@ -258,7 +284,7 @@ async def remind_submit( async def on_message( message: discord.Message, bot: commands.Bot, - cats: settings.CatCache, + cats: common.CatCache, lock: threading.Lock, logger: BoundLogger, ) -> None: @@ -276,7 +302,7 @@ async def on_message( message.channel.send( "photo from https://cataas.com/", file=discord.File( - await meow_fetch_photo(client, cats, lock), + await meow_fetch_photo(client, cats, lock, logger), description="photo from https://cataas.com/", filename="meow.png", ), diff --git a/src/bigmeow/main.py b/src/bigmeow/main.py index 022f53c..8224845 100644 --- a/src/bigmeow/main.py +++ b/src/bigmeow/main.py @@ -1,6 +1,7 @@ import asyncio import multiprocessing import signal +import threading from collections.abc import Callable from concurrent.futures import Future, ProcessPoolExecutor from dataclasses import dataclass @@ -11,18 +12,18 @@ import typer from structlog.stdlib import BoundLogger -from bigmeow import discord, scheduler, settings, telegram, web +from bigmeow import common, discord, scheduler, telegram, web from bigmeow.common import get_logger @dataclass class ShutdownHandler: - sync_store: settings.SyncStore + exit_event: threading.Event logger: BoundLogger def __call__(self, signum: int | None, frame: FrameType | None) -> None: self.logger.info("MAIN: Sending exit event to all tasks in pool") - self.sync_store.exit_event.set() + self.exit_event.set() @dataclass @@ -44,14 +45,14 @@ def __call__(self, future: Future) -> None: self.shutdown_handler(None, None) -def process_run(func, sync_store: settings.SyncStore, *arguments) -> None: +def process_run(func, sync_store: common.SyncStore, *arguments) -> None: asyncio.run(func(sync_store, *arguments)) def task_submit( run: bool, executor: ProcessPoolExecutor, - sync_store: settings.SyncStore, + sync_store: common.SyncStore, name: str, func: Callable[..., Any], shutdown_handler: ShutdownHandler, @@ -74,30 +75,29 @@ def main( logger = get_logger(__name__) manager = multiprocessing.Manager() - sync_store = settings.SyncStore( + sync_store = common.SyncStore( manager.Event(), - settings.TelegramSyncStore(manager.Queue(), manager.Queue()), - settings.DiscordSyncStore(manager.Queue()), - settings.CatCache(), + common.TelegramSyncStore(manager.Queue(), manager.Queue()), + common.DiscordSyncStore(manager.Queue()), + common.CatCache(), manager.Lock(), - settings.FactCache(), + common.FactCache(), manager.Lock(), - settings.PetrolPrice( - settings.PetrolLevel(date.min, 0, 0, 0), - settings.PetrolChange(date.min, 0, 0, 0), + common.PetrolPrice( + common.PetrolLevel(date.min, 0, 0, 0), + common.PetrolChange(date.min, 0, 0, 0), ), manager.Lock(), manager.Queue(), ) with ProcessPoolExecutor(max_workers=10) as executor: - shutdown_handler = ShutdownHandler(sync_store, logger) + shutdown_handler = ShutdownHandler(sync_store.exit_event, logger) for s in (signal.SIGHUP, signal.SIGTERM, signal.SIGINT): signal.signal(s, shutdown_handler) - foo = [] - bar = task_submit( + task_submit( run_telegram, executor, sync_store, @@ -106,9 +106,8 @@ def main( shutdown_handler, logger, ) - foo.append(bar) - bar = task_submit( + task_submit( run_discord, executor, sync_store, @@ -117,9 +116,8 @@ def main( shutdown_handler, logger, ) - foo.append(bar) - bar = task_submit( + task_submit( True, executor, sync_store, @@ -128,9 +126,8 @@ def main( shutdown_handler, logger, ) - foo.append(("s", bar)) - bar = task_submit( + task_submit( True, executor, sync_store, @@ -139,7 +136,8 @@ def main( shutdown_handler, logger, ) - foo.append(bar) + + manager.shutdown() if __name__ == "__main__": diff --git a/src/bigmeow/meow.py b/src/bigmeow/meow.py index 2102315..fa152b7 100644 --- a/src/bigmeow/meow.py +++ b/src/bigmeow/meow.py @@ -6,7 +6,7 @@ from io import BytesIO, StringIO from queue import Queue from random import choice -from typing import Any, Callable +from typing import Any, Awaitable, Callable import dateparser import httpx @@ -15,11 +15,17 @@ from structlog.stdlib import BoundLogger from bigmeow import settings -from bigmeow.common import async_lock, get_logger -from bigmeow.settings import PetrolChange, PetrolLevel, PetrolPrice +from bigmeow.common import ( + CatCache, + FactCache, + PetrolChange, + PetrolLevel, + PetrolPrice, + async_lock, +) -def meow_sayify(func: Callable) -> Callable: +def meow_sayify(func: Callable[..., Awaitable[str]]) -> Callable[..., Awaitable[str]]: async def wrapped_function(*args, **kwargs) -> str: return meow_say(await func(*args, **kwargs), wrap_text=False) @@ -28,7 +34,7 @@ async def wrapped_function(*args, **kwargs) -> str: @meow_sayify async def meow_blockedornot( - client: httpx.AsyncClient, query: str, logger: BoundLogger = get_logger(__name__) + client: httpx.AsyncClient, query: str, logger: BoundLogger ) -> str: url = "https://blockedornot.sinarproject.org/api/" @@ -68,9 +74,9 @@ def meowpetrol_update_latest( @meow_sayify async def meow_fact( client: httpx.AsyncClient, - facts: settings.FactCache, + facts: FactCache, lock: threading.Lock, - logger: BoundLogger = get_logger(__name__), + logger: BoundLogger, ) -> str: url = "https://meowfacts.herokuapp.com/" @@ -91,9 +97,9 @@ async def meow_fact( @meow_sayify async def meow_petrol( client: httpx.AsyncClient, - petrol: settings.PetrolPrice, + petrol: PetrolPrice, lock: threading.Lock, - logger: BoundLogger = get_logger(__name__), + logger: BoundLogger, ) -> str: url = "https://storage.data.gov.my/commodities/fuelprice.csv" @@ -143,9 +149,9 @@ async def meow_petrol( async def meow_fetch_photo( client: httpx.AsyncClient, - cats: settings.CatCache, + cats: CatCache, lock: threading.Lock, - logger: BoundLogger = get_logger(__name__), + logger: BoundLogger, ) -> BytesIO: url = "https://cataas.com/cat/says/meow?type=square" @@ -165,7 +171,7 @@ async def meow_prompt( message: str, channel: str, destination: str, - logger: BoundLogger = get_logger(__name__), + logger: BoundLogger, ) -> None: url = f"https://maker.ifttt.com/trigger/prompt/with/key/{settings.IFTTT_KEY}" data = {"value1": message, "value2": channel, "value3": destination} @@ -180,6 +186,7 @@ async def meow_remind( task_queue: Queue, message_queue: Queue, data_builder: Callable[[str], dict[str, Any]], + logger: BoundLogger, ) -> str: text, when = message.rsplit("@", maxsplit=1) when = dateparser.parse(when, settings={"TIMEZONE": settings.TIMEZONE.zone}) # type: ignore diff --git a/src/bigmeow/scheduler.py b/src/bigmeow/scheduler.py index 1e5a2f8..3fb0924 100644 --- a/src/bigmeow/scheduler.py +++ b/src/bigmeow/scheduler.py @@ -9,7 +9,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from structlog.stdlib import BoundLogger -from bigmeow import settings +from bigmeow import common, settings from bigmeow.common import coroutine_repeat_queue, get_logger @@ -28,7 +28,7 @@ async def task_consume( async def run( - sync_store: settings.SyncStore, logger: BoundLogger = get_logger(__name__) + sync_store: common.SyncStore, logger: BoundLogger = get_logger(__name__) ) -> None: logger.info("SCHEDULER: Starting") scheduler = AsyncIOScheduler( diff --git a/src/bigmeow/settings.py b/src/bigmeow/settings.py index 8931581..eabfcb7 100644 --- a/src/bigmeow/settings.py +++ b/src/bigmeow/settings.py @@ -1,139 +1,11 @@ -import threading -from abc import ABC from ast import literal_eval -from datetime import date -from enum import Enum -from io import BytesIO from os import environ -from queue import Queue -from random import choice, randint, shuffle -from typing import NamedTuple import pytz -import structlog -from attr import dataclass from dotenv import load_dotenv -logger = structlog.get_logger().bind(module=__name__) - load_dotenv() - -# TODO use proper typing and abstrct to abstract class in py3.12 -class CatCache: - cat_list: list[BytesIO] = [] - - def cache(self, cat: BytesIO) -> BytesIO: - logger.info("CAT_CACHE: Storing a new photo to cache") - - if len(self.cat_list) > CACHE_LIMIT: - self.cat_list[randint(0, CACHE_LIMIT - 1)] = cat - else: - self.cat_list.append(cat) - - shuffle(self.cat_list) - - return cat - - def get(self) -> BytesIO: - assert len(self.cat_list) > 0 - - logger.info("CAT_CACHE: Retrieve a photo") - return choice(self.cat_list) - - -class FactCache: - fact_list: list[str] = [] - - def cache(self, fact: str) -> str: - logger.info("FACT_CACHE: Storing a new fact to cache") - - if len(self.fact_list) > CACHE_LIMIT: - self.fact_list[randint(0, CACHE_LIMIT - 1)] = fact - else: - self.fact_list.append(fact) - - shuffle(self.fact_list) - - return fact - - def get(self) -> str: - assert len(self.fact_list) > 0 - - logger.info("FACT_CACHE: Retrieve a fact") - return choice(self.fact_list) - - -@dataclass -class PetrolRow(ABC): - date: date - ron95: float - ron97: float - diesel: float - - -class PetrolLevel(PetrolRow): - pass - - -class PetrolChange(PetrolRow): - pass - - -class PetrolPrice(NamedTuple): - level: PetrolLevel - change: PetrolChange - - -class MeowCommand(Enum): - SAY = "meowsay" - PETROL = "meowpetrol" - FACT = "meowfact" - ISBLOCKED = "meowisblocked" - THINK = "meowthink" - PROMPT = "meowprompt" - HELP = "meowhelp" - REMIND = "meowremind" - - def telegram(self) -> str: - COMMAND_PREFIX = "/" - - return f"{COMMAND_PREFIX}{self.value}" - - def __str__(self) -> str: - COMMAND_PREFIX = "!" - - return f"{COMMAND_PREFIX}{self.value}" - - -@dataclass -class TelegramSyncStore: - messages: Queue - updates: Queue - - -@dataclass -class DiscordSyncStore: - messages: Queue - - -@dataclass -class SyncStore: - exit_event: threading.Event - telegram: TelegramSyncStore - discord: DiscordSyncStore - - cats: CatCache - cat_lock: threading.Lock - - facts: FactCache - fact_lock: threading.Lock - - petrol: PetrolPrice - petrol_lock: threading.Lock - - tasks: Queue - try: DEBUG = literal_eval(environ.get("DEBUG", "False")) except Exception: diff --git a/src/bigmeow/telegram.py b/src/bigmeow/telegram.py index a0904ca..2bb3fff 100644 --- a/src/bigmeow/telegram.py +++ b/src/bigmeow/telegram.py @@ -1,6 +1,7 @@ import asyncio import json import queue +import threading from contextlib import suppress from dataclasses import dataclass from functools import partial @@ -20,8 +21,9 @@ ) from telegram.ext.filters import MessageFilter -import bigmeow.settings as settings +from bigmeow import common, settings from bigmeow.common import ( + MeowCommand, coroutine_repeat_queue, get_logger, message_contains, @@ -35,9 +37,6 @@ meow_remind, meow_say, ) -from bigmeow.settings import MeowCommand - -application = ApplicationBuilder().token(settings.TELEGRAM_TOKEN).build() @dataclass @@ -50,7 +49,9 @@ def filter(self, message: Message) -> bool: async def blockedornot_fetch( - update: Update, context: ContextTypes.DEFAULT_TYPE, logger: BoundLogger + update: Update, + context: ContextTypes.DEFAULT_TYPE, + logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing isblocked request", update=update) @@ -67,6 +68,7 @@ async def blockedornot_fetch( ) .replace(str(MeowCommand.ISBLOCKED), "") .strip(), + logger, ), reply_to_message_id=update.message.id, allow_sending_without_reply=True, @@ -77,21 +79,22 @@ async def blockedornot_fetch( def command_handler_pair( command: MeowCommand, handler: Callable[..., Any], - sync_store: settings.SyncStore, logger: BoundLogger, + **kwargs: Any, ) -> tuple[CommandHandler, MessageHandler]: return CommandHandler( - command.value, partial(handler, sync_store=sync_store, logger=logger) + command.value, partial(handler, logger=logger, **kwargs) ), MessageHandler( Filter(str(command), True), - partial(handler, sync_store=sync_store, logger=logger), + partial(handler, logger=logger, **kwargs), ) async def fact_fetch( update: Update, context: ContextTypes.DEFAULT_TYPE, - sync_store: settings.SyncStore, + facts: common.FactCache, + fact_lock: threading.Lock, logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing fact request", update=update) @@ -102,9 +105,7 @@ async def fact_fetch( context.bot.send_message( chat_id=update.effective_chat.id, parse_mode=ParseMode.MARKDOWN, - text=await meow_fact( - client, sync_store.facts, sync_store.fact_lock - ), + text=await meow_fact(client, facts, fact_lock, logger), reply_to_message_id=update.message.id, allow_sending_without_reply=True, ) @@ -114,7 +115,8 @@ async def fact_fetch( async def meow_create( update: Update, context: ContextTypes.DEFAULT_TYPE, - sync_store: settings.SyncStore, + cats: common.CatCache, + lock: threading.Lock, logger: BoundLogger, ) -> None: if not (update.message and update.effective_chat): @@ -125,9 +127,7 @@ async def meow_create( asyncio.create_task( context.bot.send_photo( chat_id=update.effective_chat.id, - photo=await meow_fetch_photo( - client, sync_store.cats, sync_store.cat_lock - ), + photo=await meow_fetch_photo(client, cats, lock, logger), caption="photo from https://cataas.com/", reply_to_message_id=update.message.id, allow_sending_without_reply=True, @@ -148,7 +148,8 @@ async def messages_consume( async def petrol_fetch( update: Update, context: ContextTypes.DEFAULT_TYPE, - sync_store: settings.SyncStore, + petrol: common.PetrolPrice, + petrol_lock: threading.Lock, logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing petrol request", update=update) @@ -159,9 +160,7 @@ async def petrol_fetch( context.bot.send_message( chat_id=update.effective_chat.id, parse_mode=ParseMode.MARKDOWN, - text=await meow_petrol( - client, sync_store.petrol, sync_store.petrol_lock - ), + text=await meow_petrol(client, petrol, petrol_lock, logger), reply_to_message_id=update.message.id, allow_sending_without_reply=True, ) @@ -169,10 +168,11 @@ async def petrol_fetch( async def run( - sync_store: settings.SyncStore, - application: Application = application, + sync_store: common.SyncStore, logger: BoundLogger = get_logger(__name__), ) -> None: + application = ApplicationBuilder().token(settings.TELEGRAM_TOKEN).build() + await setup(application, sync_store, logger) async with application: @@ -210,30 +210,47 @@ async def run( async def setup( - application: Application, sync_store: settings.SyncStore, logger: BoundLogger + application: Application, sync_store: common.SyncStore, logger: BoundLogger ) -> None: logger.info("TELEGRAM: Initializing application") application.add_handlers( list( - command_handler_pair(MeowCommand.PETROL, petrol_fetch, sync_store, logger) - + command_handler_pair(MeowCommand.SAY, say_create, sync_store, logger) - + command_handler_pair(MeowCommand.THINK, think_create, sync_store, logger) - + command_handler_pair( - MeowCommand.PROMPT, prompt_create, sync_store, logger + command_handler_pair( + MeowCommand.PETROL, + petrol_fetch, + logger, + petrol=sync_store.petrol, + petrol_lock=sync_store.petrol_lock, ) - + command_handler_pair(MeowCommand.FACT, fact_fetch, sync_store, logger) + + command_handler_pair(MeowCommand.SAY, say_create, logger) + + command_handler_pair(MeowCommand.THINK, think_create, logger) + + command_handler_pair(MeowCommand.PROMPT, prompt_create, logger) + command_handler_pair( - MeowCommand.ISBLOCKED, blockedornot_fetch, sync_store, logger + MeowCommand.FACT, + fact_fetch, + logger, + facts=sync_store.facts, + lock=sync_store.fact_lock, ) + + command_handler_pair(MeowCommand.ISBLOCKED, blockedornot_fetch, logger) + command_handler_pair( - MeowCommand.REMIND, remind_submit, sync_store, logger + MeowCommand.REMIND, + remind_submit, + logger, + tasks=sync_store.tasks, + messages=sync_store.telegram.messages, ) ) + [ MessageHandler( Filter("meow", False), - partial(meow_create, sync_store=sync_store, logger=logger), + partial( + meow_create, + logger=logger, + cats=sync_store.cats, + lock=sync_store.cat_lock, + ), ) ] ) @@ -250,7 +267,6 @@ async def setup( async def prompt_create( update: Update, _context: ContextTypes.DEFAULT_TYPE, - sync_store: settings.SyncStore, logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Dispatching prompt request", update=update) @@ -262,15 +278,15 @@ async def prompt_create( update.message.text.replace(MeowCommand.PROMPT.telegram(), "") .replace(str(MeowCommand.PROMPT), "") .strip(), - channel="telegram", - destination=json.dumps((update.effective_chat.id, update.message.id)), + "telegram", + json.dumps((update.effective_chat.id, update.message.id)), + logger, ) async def say_create( update: Update, context: ContextTypes.DEFAULT_TYPE, - sync_store: settings.SyncStore, logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing say request", update=update) @@ -294,7 +310,8 @@ async def say_create( async def remind_submit( update: Update, context: ContextTypes.DEFAULT_TYPE, - sync_store: settings.SyncStore, + tasks: queue.Queue, + messages: queue.Queue, logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing remind request", update=update) @@ -310,8 +327,8 @@ async def remind_submit( update.message.text.replace(MeowCommand.REMIND.telegram(), "") .replace(str(MeowCommand.REMIND), "") .strip(), - sync_store.tasks, - sync_store.telegram.messages, + tasks, + messages, lambda content: { "text": content, "chat_id": update.effective_chat.id, # type: ignore @@ -340,7 +357,6 @@ async def remind_submit( async def think_create( update: Update, context: ContextTypes.DEFAULT_TYPE, - sync_store: settings.SyncStore, logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing think request", update=update) diff --git a/src/bigmeow/web.py b/src/bigmeow/web.py index 862ffed..d722c03 100644 --- a/src/bigmeow/web.py +++ b/src/bigmeow/web.py @@ -10,16 +10,14 @@ from structlog.stdlib import BoundLogger from telegram.constants import ParseMode -import bigmeow.settings as settings +from bigmeow import common, settings from bigmeow.common import get_logger from bigmeow.meow import meow_say @asynccontextmanager async def lifespan(app: FastAPI): - if not hasattr(app.state, "sync_store") and isinstance( - app.state, settings.SyncStore - ): + if not hasattr(app.state, "sync_store") and isinstance(app.state, common.SyncStore): raise RuntimeError("Runtime sync_store object is missing") yield @@ -64,7 +62,7 @@ def check_login_is_valid(authorization: str | None) -> bool: async def run( - sync_store: settings.SyncStore, logger: BoundLogger = get_logger(__name__) + sync_store: common.SyncStore, logger: BoundLogger = get_logger(__name__) ) -> None: app.state.sync_store = sync_store From 06d0a1d333462cf60e08209b203e972d9ff79431 Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Wed, 4 Jun 2025 14:06:46 +0800 Subject: [PATCH 4/9] fix wrong dataclass import --- README.md | 9 +-------- src/bigmeow/common.py | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 419304b..ac4a907 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,6 @@ TELEGRAM_USER= WEBHOOK_URL= DEBUG= IFTTT_KEY= -SLACK_SECRET_SIGN= -SLACK_SECRET_CLIENT= -SLACK_CLIENT_ID= DATABASE_URL=postgresql+psycopg://:@:/ ``` @@ -58,9 +55,5 @@ $ poetry run python -m bigmeow.main Optionally, you can run the application with the following options: -* `--noweb` to run without a web frontend * `--notg` to run without a telegram bot -* `--nodiscord` to run without a discord bot -* `--noslack` to run without a slack bot - -Notice slack and telegram bot would fail to receive event input without the web frontend \ No newline at end of file +* `--nodiscord` to run without a discord bot \ No newline at end of file diff --git a/src/bigmeow/common.py b/src/bigmeow/common.py index 0a837bb..d358ea3 100644 --- a/src/bigmeow/common.py +++ b/src/bigmeow/common.py @@ -4,6 +4,7 @@ from abc import ABC from collections.abc import Callable from contextlib import asynccontextmanager +from dataclasses import dataclass from datetime import date from enum import Enum from io import BytesIO @@ -12,7 +13,6 @@ from typing import Any, Awaitable, NamedTuple import structlog -from attr import dataclass from bigmeow import settings From 225b3c283899f06bcd910fa01fe498e375e3cd0f Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Wed, 4 Jun 2025 23:41:33 +0800 Subject: [PATCH 5/9] crude test for discord --- src/bigmeow/discord.py | 113 ++++++++++----------- src/bigmeow/telegram.py | 8 +- tests/conftest.py | 16 +++ tests/test_discord.py | 218 ++++++++++++++++++++++++++++++++++++++++ tests/test_meow.py | 59 +++++++++++ 5 files changed, 353 insertions(+), 61 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_discord.py create mode 100644 tests/test_meow.py diff --git a/src/bigmeow/discord.py b/src/bigmeow/discord.py index 7df373d..a8de405 100644 --- a/src/bigmeow/discord.py +++ b/src/bigmeow/discord.py @@ -6,7 +6,8 @@ from functools import partial from io import StringIO from multiprocessing.synchronize import Event as Event -from typing import Any +from types import SimpleNamespace +from typing import Any, Callable import discord import httpx @@ -116,6 +117,12 @@ def setup(bot: commands.Bot, sync_store: common.SyncStore, logger: BoundLogger) ) +def command_make( + command: MeowCommand, func: Callable[..., Any], logger: BoundLogger, **kwargs: Any +): + return commands.command(command.value, extras=dict(logger=logger, **kwargs))(func) + + async def messages_consume( bot: commands.Bot, messages: queue.Queue, logger: BoundLogger ) -> None: @@ -144,36 +151,28 @@ async def messages_consume( asyncio.create_task(text_send(data["content"], channel, message)) -def command_make(command: MeowCommand, func, logger: BoundLogger, **kwargs: Any): - @commands.command(command.value) - async def inner(*args_inner, **kwargs_inner): - return await func(*args_inner, **kwargs_inner, **kwargs, logger=logger) - - return inner - +async def petrol_fetch(context: commands.Context) -> None: + assert context.command -async def petrol_fetch( - context: commands.Context, - petrol: common.PetrolPrice, - lock: threading.Lock, - logger: BoundLogger, -) -> None: - logger.info("DISCORD: Received a command", message=context.message) + extras = SimpleNamespace(**context.command.extras) + extras.logger.info("DISCORD: Received a command", message=context.message) async with httpx.AsyncClient() as client: asyncio.create_task( text_send( - await meow_petrol(client, petrol, lock, logger), + await meow_petrol( + client, + extras.petrol, + extras.lock, + extras.logger, + ), context.message.channel, context.message, ) ) -async def say_create( - context: commands.Context, - *args: str, - logger: BoundLogger, -) -> None: + +async def say_create(context: commands.Context, *args: str) -> None: asyncio.create_task( text_send( meow_say(" ".join(args).strip()), @@ -183,26 +182,22 @@ async def say_create( ) -async def prompt_create( - context: commands.Context, - *args: str, - logger: BoundLogger, -) -> None: +async def prompt_create(context: commands.Context, *args: str) -> None: + assert context.command + + extras = SimpleNamespace(**context.command.extras) + async with httpx.AsyncClient() as client: await meow_prompt( client, " ".join(args).strip(), channel="discord", destination=json.dumps((context.message.channel.id, context.message.id)), - logger=logger, + logger=extras.logger, ) -async def think_create( - context: commands.Context, - *args: str, - logger: BoundLogger, -) -> None: +async def think_create(context: commands.Context, *args: str) -> None: asyncio.create_task( text_send( meow_say(" ".join(args).strip(), is_cowthink=True), @@ -212,66 +207,68 @@ async def think_create( ) -async def blockedornot_fetch( - context: commands.Context, - url: str, - logger: BoundLogger, -) -> None: +async def blockedornot_fetch(context: commands.Context, url: str) -> None: + assert context.command + + extras = SimpleNamespace(**context.command.extras) + async with httpx.AsyncClient() as client: asyncio.create_task( text_send( - await meow_blockedornot(client, url, logger), + await meow_blockedornot(client, url, extras.logger), context.message.channel, context.message, ) ) -async def fact_fetch( - context: commands.Context, - facts: common.FactCache, - lock: threading.Lock, - logger: BoundLogger, -) -> None: +async def fact_fetch(context: commands.Context) -> None: + assert context.command + + extras = SimpleNamespace(**context.command.extras) + async with httpx.AsyncClient() as client: asyncio.create_task( text_send( - await meow_fact(client, facts, lock, logger), + await meow_fact( + client, + extras.facts, + extras.lock, + extras.logger, + ), context.message.channel, context.message, ) ) -async def remind_submit( - context: commands.Context, - *args: str, - tasks: queue.Queue, - messages: queue.Queue, - logger: BoundLogger, -) -> None: - logger.info("DISCORD: Processing remind request", message=context.message) +async def remind_submit(context: commands.Context, *args: str) -> None: + assert context.command + + extras = SimpleNamespace(**context.command.extras) + extras.logger.info("DISCORD: Processing remind request", message=context.message) try: asyncio.create_task( text_send( await meow_remind( " ".join(args).strip(), - tasks, - messages, + extras.tasks, + extras.messages, lambda content: { "content": content, "channel_id": context.message.channel.id, "message_id": context.message.id, }, - logger, + extras.logger, ), context.message.channel, context.message, ) ) - except (ValueError, AssertionError): + except (ValueError, AssertionError) as e: + extras.logger.exception(e) # type: ignore asyncio.create_task( text_send( "Fail to schedule message, please check format again", @@ -328,4 +325,4 @@ async def text_send( else {"content": content} ), ) - ) \ No newline at end of file + ) diff --git a/src/bigmeow/telegram.py b/src/bigmeow/telegram.py index 2bb3fff..04c3929 100644 --- a/src/bigmeow/telegram.py +++ b/src/bigmeow/telegram.py @@ -94,7 +94,7 @@ async def fact_fetch( update: Update, context: ContextTypes.DEFAULT_TYPE, facts: common.FactCache, - fact_lock: threading.Lock, + lock: threading.Lock, logger: BoundLogger, ) -> None: logger.info("TELEGRAM: Processing fact request", update=update) @@ -105,7 +105,7 @@ async def fact_fetch( context.bot.send_message( chat_id=update.effective_chat.id, parse_mode=ParseMode.MARKDOWN, - text=await meow_fact(client, facts, fact_lock, logger), + text=await meow_fact(client, facts, lock, logger), reply_to_message_id=update.message.id, allow_sending_without_reply=True, ) @@ -336,13 +336,15 @@ async def remind_submit( "reply_to_message_id": update.message.id, # type: ignore "allow_sending_without_reply": True, }, + logger, ), reply_to_message_id=update.message.id, allow_sending_without_reply=True, ) ) - except (ValueError, AssertionError): + except (ValueError, AssertionError) as e: + logger.exception(e) # type: ignore asyncio.create_task( context.bot.send_message( chat_id=update.effective_chat.id, # type: ignore diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6d12825 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from structlog.stdlib import BoundLogger + + +@pytest.fixture(name="logger") +def logger_fixture(): + yield MagicMock(spec=BoundLogger) + + +@pytest.fixture(name="HttpxClient") +def httpx_client(): + with patch("bigmeow.discord.httpx.AsyncClient", spec=httpx.AsyncClient) as Client: + yield Client diff --git a/tests/test_discord.py b/tests/test_discord.py new file mode 100644 index 0000000..ce0bc90 --- /dev/null +++ b/tests/test_discord.py @@ -0,0 +1,218 @@ +import json +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import discord +import pytest +from discord.ext import commands +from structlog.stdlib import BoundLogger + +from bigmeow.discord import ( + blockedornot_fetch, + fact_fetch, + petrol_fetch, + prompt_create, + remind_submit, + say_create, + think_create, +) + + +@pytest.fixture(name="context") +def context_fixture(): + context = MagicMock(commands.Context) + context.command = MagicMock(commands.Command) + context.message = MagicMock(discord.Message) + context.message.channel = MagicMock(discord.TextChannel) + + yield context + + +@pytest.fixture(name="text_send") +def text_send_fixture(): + with patch("bigmeow.discord.text_send") as text_send: + yield text_send + + +@pytest.mark.asyncio +async def test_petrol( + logger: BoundLogger, + context: commands.Context, + text_send: AsyncMock, + HttpxClient: Any, +): + with ( + patch("bigmeow.discord.meow_petrol") as meow_petrol, + ): + meow_petrol.return_value = "petrol report" + + assert context.command + context.command.extras = { + "logger": logger, + "petrol": MagicMock(), + "lock": MagicMock(), + } + + await petrol_fetch(context) + + HttpxClient.assert_called_once() + + async with HttpxClient() as client: + meow_petrol.assert_awaited_once_with( + client, + context.command.extras["petrol"], + context.command.extras["lock"], + logger, + ) + + text_send.assert_called_once_with( + meow_petrol.return_value, context.message.channel, context.message + ) + + +@pytest.mark.asyncio +async def test_say_create( + logger: BoundLogger, context: commands.Context, text_send: AsyncMock +): + with patch("bigmeow.discord.meow_say") as meow_say: + meow_say.return_value = "decorated" + + await say_create(context, "hello", "world") + + meow_say.assert_called_once_with("hello world") + + text_send.assert_called_once_with( + meow_say.return_value, context.message.channel, context.message + ) + + +@pytest.mark.asyncio +async def test_think_create( + logger: BoundLogger, context: commands.Context, text_send: AsyncMock +): + with patch("bigmeow.discord.meow_say") as meow_say: + meow_say.return_value = "decorated" + + await think_create(context, "hello", "world") + + meow_say.assert_called_once_with("hello world", is_cowthink=True) + + text_send.assert_called_once_with( + meow_say.return_value, context.message.channel, context.message + ) + + +@pytest.mark.asyncio +async def test_prompt_create( + logger: BoundLogger, context: commands.Context, HttpxClient: Any +): + with patch("bigmeow.discord.meow_prompt") as meow_prompt: + assert context.command + context.command.extras = {"logger": logger} + + context.message.channel.id = 1 + context.message.id = 1 + + await prompt_create(context, "le", "petit", "prince") + + async with HttpxClient() as client: + meow_prompt.assert_called_once_with( + client, + "le petit prince", + channel="discord", + destination=json.dumps( + (context.message.channel.id, context.message.id) + ), + logger=logger, + ) + + +@pytest.mark.asyncio +async def test_blockedornot( + logger: BoundLogger, + context: commands.Context, + HttpxClient: Any, + text_send: AsyncMock, +): + with patch("bigmeow.discord.meow_blockedornot") as meow_blockedornot: + meow_blockedornot.return_value = "result" + + assert context.command + context.command.extras = {"logger": logger} + + await blockedornot_fetch(context, "example.org") + + async with HttpxClient() as client: + meow_blockedornot.assert_awaited_once_with(client, "example.org", logger) + + text_send.assert_called_once_with( + "result", context.message.channel, context.message + ) + + +@pytest.mark.asyncio +async def test_fact_fetch( + logger: BoundLogger, + context: commands.Context, + HttpxClient: Any, + text_send: AsyncMock, +): + with patch("bigmeow.discord.meow_fact") as meow_fact: + meow_fact.return_value = "fact" + + assert context.command + context.command.extras = { + "logger": logger, + "facts": MagicMock(), + "lock": MagicMock(), + } + + await fact_fetch(context) + + async with HttpxClient() as client: + meow_fact.assert_awaited_once_with( + client, + context.command.extras["facts"], + context.command.extras["lock"], + logger, + ) + + text_send.assert_called_once_with( + "fact", context.message.channel, context.message + ) + + +@pytest.mark.asyncio +async def test_remind_submit( + logger: BoundLogger, context: commands.Context, text_send: AsyncMock +): + with patch("bigmeow.discord.meow_remind") as meow_remind: + meow_remind.return_value = "scheduled" + + assert context.command + context.command.extras = { + "logger": logger, + "tasks": MagicMock(), + "messages": MagicMock(), + } + + await remind_submit(context, "do", "this", "@", "5", "seconds", "later") + + meow_remind.assert_awaited_once() + + assert "do this @ 5 seconds later" == meow_remind.call_args.args[0] + assert context.command.extras["tasks"] == meow_remind.call_args.args[1] + assert context.command.extras["messages"] == meow_remind.call_args.args[2] + assert logger == meow_remind.call_args.args[4] + + result = meow_remind.call_args.args[3]("do this") + expected = { + "content": "do this", + "channel_id": context.message.channel.id, + "message_id": context.message.id, + } + assert expected == result + + text_send.assert_called_once_with( + "scheduled", context.message.channel, context.message + ) diff --git a/tests/test_meow.py b/tests/test_meow.py new file mode 100644 index 0000000..1651be4 --- /dev/null +++ b/tests/test_meow.py @@ -0,0 +1,59 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +import pytest_asyncio +from structlog.stdlib import BoundLogger + +from bigmeow.meow import meow_blockedornot + + +@pytest_asyncio.fixture(name="httpx_client") +async def httpx_client_fixture(): + with patch("httpx.AsyncClient") as client: + client.get = AsyncMock() + client.post = AsyncMock() + + yield client + + +@pytest.mark.asyncio +async def test_blockedornot(httpx_client: httpx.AsyncClient, logger: BoundLogger): + response = MagicMock(spec=httpx.Response) + response.json.return_value = { + "blocked": True, + "different_ip": True, + "measurement": "https://example.org", + } + httpx_client.get.return_value = response + + result = await meow_blockedornot(httpx_client, "example.com", logger) + + assert "is blocked" in result + assert "blockedornot.sinarproject.org" in result + + response = MagicMock(spec=httpx.Response) + response.json.return_value = { + "blocked": False, + "different_ip": True, + "measurement": "https://example.org", + } + httpx_client.get.return_value = response + + result = await meow_blockedornot(httpx_client, "example.net", logger) + + assert "likely safe" in result + assert "blockedornot.sinarproject.org" in result + + response = MagicMock(spec=httpx.Response) + response.json.return_value = { + "blocked": False, + "different_ip": False, + "measurement": "https://example.org", + } + httpx_client.get.return_value = response + + result = await meow_blockedornot(httpx_client, "example.info", logger) + + assert "is safe" in result + assert "blockedornot.sinarproject.org" in result From 269b9d58ac960cb79cc50009fcf6f8063069514f Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Fri, 6 Jun 2025 19:37:52 +0800 Subject: [PATCH 6/9] use asyncio executor instead --- src/bigmeow/common.py | 17 ++++++++------ src/bigmeow/discord.py | 35 ++++++++++++++++++++-------- src/bigmeow/main.py | 1 + src/bigmeow/meow.py | 18 +++++++-------- src/bigmeow/scheduler.py | 38 ++++++++++++++++++++++++++---- src/bigmeow/telegram.py | 43 +++++++++++++++++++++++----------- src/bigmeow/web.py | 50 ++++++++++++++++++++++------------------ tests/test_discord.py | 32 ++++++++++++------------- 8 files changed, 150 insertions(+), 84 deletions(-) diff --git a/src/bigmeow/common.py b/src/bigmeow/common.py index d358ea3..745d3bb 100644 --- a/src/bigmeow/common.py +++ b/src/bigmeow/common.py @@ -1,6 +1,5 @@ import asyncio import logging -import threading from abc import ABC from collections.abc import Callable from contextlib import asynccontextmanager @@ -8,8 +7,10 @@ from datetime import date from enum import Enum from io import BytesIO +from multiprocessing.managers import ListProxy from queue import Queue from random import choice, randint, shuffle +from threading import Event, Lock from typing import Any, Awaitable, NamedTuple import structlog @@ -127,24 +128,25 @@ class DiscordSyncStore: @dataclass class SyncStore: - exit_event: threading.Event + exit_event: Event telegram: TelegramSyncStore discord: DiscordSyncStore cats: CatCache - cat_lock: threading.Lock + cat_lock: Lock facts: FactCache - fact_lock: threading.Lock + fact_lock: Lock petrol: PetrolPrice - petrol_lock: threading.Lock + petrol_lock: Lock tasks: Queue + scheduled: ListProxy @asynccontextmanager -async def async_lock(lock: threading.Lock): +async def async_lock(lock: Lock): await asyncio.to_thread(lock.acquire) try: @@ -158,6 +160,7 @@ def message_contains(message: str | None, content: str, is_command=True) -> bool return (message.startswith(content)) if is_command else (content in message.lower()) + async def coroutine_repeat_queue( coro_func: Callable[..., Awaitable[None]], *args: Any ) -> None: @@ -165,4 +168,4 @@ async def coroutine_repeat_queue( while True: await coro_func(*args) except asyncio.CancelledError: - pass \ No newline at end of file + pass diff --git a/src/bigmeow/discord.py b/src/bigmeow/discord.py index a8de405..1bffd98 100644 --- a/src/bigmeow/discord.py +++ b/src/bigmeow/discord.py @@ -123,6 +123,21 @@ def command_make( return commands.command(command.value, extras=dict(logger=logger, **kwargs))(func) +async def message_produce( + message: str, queue: queue.Queue, channel_id, message_id, logger: BoundLogger +): + asyncio.create_task( + asyncio.to_thread( + queue.put, + { + "content": meow_say(message), + "channel_id": channel_id, + "message_id": message_id, + }, + ) + ) + + async def messages_consume( bot: commands.Bot, messages: queue.Queue, logger: BoundLogger ) -> None: @@ -251,16 +266,16 @@ async def remind_submit(context: commands.Context, *args: str) -> None: try: asyncio.create_task( text_send( - await meow_remind( - " ".join(args).strip(), - extras.tasks, - extras.messages, - lambda content: { - "content": content, - "channel_id": context.message.channel.id, - "message_id": context.message.id, - }, - extras.logger, + meow_say( + await meow_remind( + " ".join(args).strip(), + extras.tasks, + extras.logger, + message_produce, + extras.messages, + context.message.channel.id, + context.message.id, + ) ), context.message.channel, context.message, diff --git a/src/bigmeow/main.py b/src/bigmeow/main.py index 8224845..5a1b2b0 100644 --- a/src/bigmeow/main.py +++ b/src/bigmeow/main.py @@ -89,6 +89,7 @@ def main( ), manager.Lock(), manager.Queue(), + manager.list() ) with ProcessPoolExecutor(max_workers=10) as executor: diff --git a/src/bigmeow/meow.py b/src/bigmeow/meow.py index fa152b7..792704f 100644 --- a/src/bigmeow/meow.py +++ b/src/bigmeow/meow.py @@ -1,12 +1,13 @@ import asyncio import csv import threading +from collections.abc import Awaitable, Callable from datetime import date, timedelta from functools import reduce from io import BytesIO, StringIO from queue import Queue from random import choice -from typing import Any, Awaitable, Callable +from typing import Any import dateparser import httpx @@ -184,9 +185,9 @@ async def meow_prompt( async def meow_remind( message: str, task_queue: Queue, - message_queue: Queue, - data_builder: Callable[[str], dict[str, Any]], logger: BoundLogger, + func: Callable[..., Awaitable[Any]], + *args: Any, ) -> str: text, when = message.rsplit("@", maxsplit=1) when = dateparser.parse(when, settings={"TIMEZONE": settings.TIMEZONE.zone}) # type: ignore @@ -196,17 +197,16 @@ async def meow_remind( await asyncio.to_thread( task_queue.put, { - "func": "bigmeow.scheduler:execute_sync", + "func": func, + "name": message, "trigger": DateTrigger(when, settings.TIMEZONE), - "args": ( - message_queue.put, - data_builder(meow_say(text)), - ), + "args": (text, *args), + "kwargs": {"logger": logger}, "misfire_grace_time": None, }, ) - return f"Scheduled message: {text}\nTime: {when}" + return f"Scheduled message: {text}\n\nTime: {when}" def meow_say(message: str, is_cowthink: bool = False, wrap_text: bool = True) -> str: diff --git a/src/bigmeow/scheduler.py b/src/bigmeow/scheduler.py index 3fb0924..978e728 100644 --- a/src/bigmeow/scheduler.py +++ b/src/bigmeow/scheduler.py @@ -1,10 +1,13 @@ import asyncio from collections.abc import Callable from contextlib import suppress +from dataclasses import dataclass +from multiprocessing.managers import ListProxy from queue import Empty, Queue from typing import Any -from apscheduler.executors.pool import ProcessPoolExecutor +from apscheduler import events +from apscheduler.executors.asyncio import AsyncIOExecutor from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore from apscheduler.schedulers.asyncio import AsyncIOScheduler from structlog.stdlib import BoundLogger @@ -13,8 +16,21 @@ from bigmeow.common import coroutine_repeat_queue, get_logger -def execute_sync(callable: Callable[..., Any], *args) -> None: - callable(*args) +@dataclass +class ScheduledUpdater: + scheduler: AsyncIOScheduler + event_loop: asyncio.AbstractEventLoop + scheduled: ListProxy + lock: asyncio.Lock + logger: BoundLogger + + def __call__(self, event): + self.event_loop.create_task(self.run(event)) + + async def run(self, event): + self.logger.info("Updating jobs", on_event=event) + async with self.lock: + self.scheduled[:] = self.scheduler.get_jobs() async def task_consume( @@ -37,7 +53,19 @@ async def run( jobstores={ settings.TASK_DEFAULT_STORE: SQLAlchemyJobStore(settings.DATABASE_URL) }, - executors={settings.TASK_DEFAULT_EXECUTOR: ProcessPoolExecutor(10)}, + executors={ + settings.TASK_DEFAULT_EXECUTOR: AsyncIOExecutor(), + }, + ) + scheduler.add_listener( + ScheduledUpdater( + scheduler, + asyncio.get_running_loop(), + sync_store.scheduled, + asyncio.Lock(), + logger, + ), + events.EVENT_ALL, ) scheduler.start() @@ -50,4 +78,4 @@ async def run( logger.info("SCHEDULER: Stopping") - scheduler.shutdown() \ No newline at end of file + scheduler.shutdown() diff --git a/src/bigmeow/telegram.py b/src/bigmeow/telegram.py index 04c3929..f07a8bf 100644 --- a/src/bigmeow/telegram.py +++ b/src/bigmeow/telegram.py @@ -135,6 +135,23 @@ async def meow_create( ) +async def message_produce( + message: str, queue, chat_id, message_id, logger: BoundLogger +): + asyncio.create_task( + asyncio.to_thread( + queue.put, + { + "text": meow_say(message), + "chat_id": chat_id, + "parse_mode": ParseMode.MARKDOWN, + "reply_to_message_id": message_id, + "allow_sending_without_reply": True, + }, + ) + ) + + async def messages_consume( application: Application, messages: queue.Queue, logger: BoundLogger ) -> None: @@ -323,20 +340,18 @@ async def remind_submit( context.bot.send_message( chat_id=update.effective_chat.id, parse_mode=ParseMode.MARKDOWN, - text=await meow_remind( - update.message.text.replace(MeowCommand.REMIND.telegram(), "") - .replace(str(MeowCommand.REMIND), "") - .strip(), - tasks, - messages, - lambda content: { - "text": content, - "chat_id": update.effective_chat.id, # type: ignore - "parse_mode": ParseMode.MARKDOWN, - "reply_to_message_id": update.message.id, # type: ignore - "allow_sending_without_reply": True, - }, - logger, + text=meow_say( + await meow_remind( + update.message.text.replace(MeowCommand.REMIND.telegram(), "") + .replace(str(MeowCommand.REMIND), "") + .strip(), + tasks, + logger, + message_produce, + messages, + update.effective_chat.id, + update.message.id, + ) ), reply_to_message_id=update.message.id, allow_sending_without_reply=True, diff --git a/src/bigmeow/web.py b/src/bigmeow/web.py index d722c03..21d9a73 100644 --- a/src/bigmeow/web.py +++ b/src/bigmeow/web.py @@ -8,11 +8,9 @@ from fastapi import Depends, FastAPI, Header, Request from fastapi.responses import PlainTextResponse from structlog.stdlib import BoundLogger -from telegram.constants import ParseMode -from bigmeow import common, settings +from bigmeow import common, discord, settings, telegram from bigmeow.common import get_logger -from bigmeow.meow import meow_say @asynccontextmanager @@ -131,6 +129,20 @@ async def telegram_webhook( ) +@app.get("/api/scheduled") +async def scheduled(request: Request, logger: BoundLogger = Depends(Logger())): + logger.info(request.app.state.sync_store.scheduled) + return [ + { + "id": job.id, + "name": job.name, + "executor": job.executor, + "when": job.next_run_time, + } + for job in request.app.state.sync_store.scheduled + ] + + @app.post(settings.ECHO_WEBHOOK, include_in_schema=False) async def chat_post( request: Request, @@ -138,6 +150,7 @@ async def chat_post( x_destination: Annotated[str, Header()], logger: BoundLogger = Depends(Logger()), ) -> None: + # FIXME need auth text = (await request.body()).decode() logger.info( @@ -148,33 +161,24 @@ async def chat_post( ) match x_channel: case "telegram": - chat_id, message_id = json.loads(x_destination) - asyncio.create_task( - asyncio.to_thread( - request.app.state.sync_store.telegram.messages.put, - { - "text": meow_say(text), - "chat_id": chat_id, - "parse_mode": ParseMode.MARKDOWN, - "reply_to_message_id": message_id, - "allow_sending_without_reply": True, - }, + telegram.message_produce( + text, + request.app.state.sync_store.telegram.messages, + *json.loads(x_destination), + logger=logger, ) ) case "discord": - channel_id, message_id = json.loads(x_destination) asyncio.create_task( - asyncio.to_thread( - request.app.state.sync_store.discord.messages.put, - { - "content": meow_say(text), - "channel_id": channel_id, - "message_id": message_id, - }, + discord.message_produce( + text, + request.app.state.sync_store.discord.messages, + *json.loads(x_destination), + logger=logger, ) ) case _: - raise Exception("Invalid channel") \ No newline at end of file + raise Exception("Invalid channel") diff --git a/tests/test_discord.py b/tests/test_discord.py index ce0bc90..9569743 100644 --- a/tests/test_discord.py +++ b/tests/test_discord.py @@ -10,6 +10,7 @@ from bigmeow.discord import ( blockedornot_fetch, fact_fetch, + message_produce, petrol_fetch, prompt_create, remind_submit, @@ -186,8 +187,12 @@ async def test_fact_fetch( async def test_remind_submit( logger: BoundLogger, context: commands.Context, text_send: AsyncMock ): - with patch("bigmeow.discord.meow_remind") as meow_remind: + with ( + patch("bigmeow.discord.meow_remind") as meow_remind, + patch("bigmeow.discord.meow_say") as meow_say, + ): meow_remind.return_value = "scheduled" + meow_say.return_value = "meowed scheduled" assert context.command context.command.extras = { @@ -198,21 +203,16 @@ async def test_remind_submit( await remind_submit(context, "do", "this", "@", "5", "seconds", "later") - meow_remind.assert_awaited_once() - - assert "do this @ 5 seconds later" == meow_remind.call_args.args[0] - assert context.command.extras["tasks"] == meow_remind.call_args.args[1] - assert context.command.extras["messages"] == meow_remind.call_args.args[2] - assert logger == meow_remind.call_args.args[4] - - result = meow_remind.call_args.args[3]("do this") - expected = { - "content": "do this", - "channel_id": context.message.channel.id, - "message_id": context.message.id, - } - assert expected == result + meow_remind.assert_awaited_once_with( + "do this @ 5 seconds later", + context.command.extras["tasks"], + context.command.extras["logger"], + message_produce, + context.command.extras["messages"], + context.message.channel.id, + context.message.id, + ) text_send.assert_called_once_with( - "scheduled", context.message.channel, context.message + meow_say("scheduled"), context.message.channel, context.message ) From f1924d26c6cad7fb08ac9c82de9ccfbc80d168c1 Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Thu, 19 Jun 2025 15:52:04 +0800 Subject: [PATCH 7/9] Add auth for echo request --- README.md | 15 ++++- poetry.lock | 123 +++++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 + src/bigmeow/settings.py | 1 + src/bigmeow/web.py | 8 ++- 5 files changed, 143 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ac4a907..7ef615c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ A dumb experimental bot done for no good reason * `!meowisblocked domain.tld`: Perform a query to https://blockedornot.sinarproject.org/ to check if a domain is blocked in Malaysia * `!meowprompt prompt`: a reply to the supplied prompt, powered by https://ifttt.com/ * `!meowremind text @ when`: Set a reminder for yourself after the time specified -* If your message has a `meow` in it, a cat photo is fetched from https://cataas.com/ (Currently not supported on slack) +* If your message has a `meow` in it, a cat photo is fetched from https://cataas.com/ NOTE: all `!` commands can be replaced by `/` in telegram, e.g. `/meowsay hello world`. @@ -27,15 +27,24 @@ NOTE: all `!` commands can be replaced by `/` in telegram, e.g. `/meowsay hello You can pull an image from https://hub.docker.com/r/jeffrey04/bigmeow_bot and supply the following environment variables to run the container. ``` -DISCORD_APP_ID= -DISCORD_APP_PUBLIC= DISCORD_TOKEN= DISCORD_USER= + TELEGRAM_TOKEN= TELEGRAM_USER= +TELEGRAM_WEB_TOKEN= + +ECHO_TOKEN= + WEBHOOK_URL= + +WEB_SECRET_PING= +WEB_EECRET_PASSWORD= + DEBUG= + IFTTT_KEY= + DATABASE_URL=postgresql+psycopg://:@:/ ``` diff --git a/poetry.lock b/poetry.lock index 4873187..41080a0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -394,6 +394,89 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coverage" +version = "7.8.2" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "coverage-7.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bd8ec21e1443fd7a447881332f7ce9d35b8fbd2849e761bb290b584535636b0a"}, + {file = "coverage-7.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c26c2396674816deaeae7ded0e2b42c26537280f8fe313335858ffff35019be"}, + {file = "coverage-7.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1aec326ed237e5880bfe69ad41616d333712c7937bcefc1343145e972938f9b3"}, + {file = "coverage-7.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e818796f71702d7a13e50c70de2a1924f729228580bcba1607cccf32eea46e6"}, + {file = "coverage-7.8.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:546e537d9e24efc765c9c891328f30f826e3e4808e31f5d0f87c4ba12bbd1622"}, + {file = "coverage-7.8.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ab9b09a2349f58e73f8ebc06fac546dd623e23b063e5398343c5270072e3201c"}, + {file = "coverage-7.8.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fd51355ab8a372d89fb0e6a31719e825cf8df8b6724bee942fb5b92c3f016ba3"}, + {file = "coverage-7.8.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0774df1e093acb6c9e4d58bce7f86656aeed6c132a16e2337692c12786b32404"}, + {file = "coverage-7.8.2-cp310-cp310-win32.whl", hash = "sha256:00f2e2f2e37f47e5f54423aeefd6c32a7dbcedc033fcd3928a4f4948e8b96af7"}, + {file = "coverage-7.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:145b07bea229821d51811bf15eeab346c236d523838eda395ea969d120d13347"}, + {file = "coverage-7.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b99058eef42e6a8dcd135afb068b3d53aff3921ce699e127602efff9956457a9"}, + {file = "coverage-7.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5feb7f2c3e6ea94d3b877def0270dff0947b8d8c04cfa34a17be0a4dc1836879"}, + {file = "coverage-7.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:670a13249b957bb9050fab12d86acef7bf8f6a879b9d1a883799276e0d4c674a"}, + {file = "coverage-7.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bdc8bf760459a4a4187b452213e04d039990211f98644c7292adf1e471162b5"}, + {file = "coverage-7.8.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07a989c867986c2a75f158f03fdb413128aad29aca9d4dbce5fc755672d96f11"}, + {file = "coverage-7.8.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2db10dedeb619a771ef0e2949ccba7b75e33905de959c2643a4607bef2f3fb3a"}, + {file = "coverage-7.8.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e6ea7dba4e92926b7b5f0990634b78ea02f208d04af520c73a7c876d5a8d36cb"}, + {file = "coverage-7.8.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ef2f22795a7aca99fc3c84393a55a53dd18ab8c93fb431004e4d8f0774150f54"}, + {file = "coverage-7.8.2-cp311-cp311-win32.whl", hash = "sha256:641988828bc18a6368fe72355df5f1703e44411adbe49bba5644b941ce6f2e3a"}, + {file = "coverage-7.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8ab4a51cb39dc1933ba627e0875046d150e88478dbe22ce145a68393e9652975"}, + {file = "coverage-7.8.2-cp311-cp311-win_arm64.whl", hash = "sha256:8966a821e2083c74d88cca5b7dcccc0a3a888a596a04c0b9668a891de3a0cc53"}, + {file = "coverage-7.8.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e2f6fe3654468d061942591aef56686131335b7a8325684eda85dacdf311356c"}, + {file = "coverage-7.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76090fab50610798cc05241bf83b603477c40ee87acd358b66196ab0ca44ffa1"}, + {file = "coverage-7.8.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bd0a0a5054be160777a7920b731a0570284db5142abaaf81bcbb282b8d99279"}, + {file = "coverage-7.8.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da23ce9a3d356d0affe9c7036030b5c8f14556bd970c9b224f9c8205505e3b99"}, + {file = "coverage-7.8.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9392773cffeb8d7e042a7b15b82a414011e9d2b5fdbbd3f7e6a6b17d5e21b20"}, + {file = "coverage-7.8.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:876cbfd0b09ce09d81585d266c07a32657beb3eaec896f39484b631555be0fe2"}, + {file = "coverage-7.8.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3da9b771c98977a13fbc3830f6caa85cae6c9c83911d24cb2d218e9394259c57"}, + {file = "coverage-7.8.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a990f6510b3292686713bfef26d0049cd63b9c7bb17e0864f133cbfd2e6167f"}, + {file = "coverage-7.8.2-cp312-cp312-win32.whl", hash = "sha256:bf8111cddd0f2b54d34e96613e7fbdd59a673f0cf5574b61134ae75b6f5a33b8"}, + {file = "coverage-7.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:86a323a275e9e44cdf228af9b71c5030861d4d2610886ab920d9945672a81223"}, + {file = "coverage-7.8.2-cp312-cp312-win_arm64.whl", hash = "sha256:820157de3a589e992689ffcda8639fbabb313b323d26388d02e154164c57b07f"}, + {file = "coverage-7.8.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ea561010914ec1c26ab4188aef8b1567272ef6de096312716f90e5baa79ef8ca"}, + {file = "coverage-7.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cb86337a4fcdd0e598ff2caeb513ac604d2f3da6d53df2c8e368e07ee38e277d"}, + {file = "coverage-7.8.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a4636ddb666971345541b59899e969f3b301143dd86b0ddbb570bd591f1e85"}, + {file = "coverage-7.8.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5040536cf9b13fb033f76bcb5e1e5cb3b57c4807fef37db9e0ed129c6a094257"}, + {file = "coverage-7.8.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc67994df9bcd7e0150a47ef41278b9e0a0ea187caba72414b71dc590b99a108"}, + {file = "coverage-7.8.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e6c86888fd076d9e0fe848af0a2142bf606044dc5ceee0aa9eddb56e26895a0"}, + {file = "coverage-7.8.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:684ca9f58119b8e26bef860db33524ae0365601492e86ba0b71d513f525e7050"}, + {file = "coverage-7.8.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8165584ddedb49204c4e18da083913bdf6a982bfb558632a79bdaadcdafd0d48"}, + {file = "coverage-7.8.2-cp313-cp313-win32.whl", hash = "sha256:34759ee2c65362163699cc917bdb2a54114dd06d19bab860725f94ef45a3d9b7"}, + {file = "coverage-7.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:2f9bc608fbafaee40eb60a9a53dbfb90f53cc66d3d32c2849dc27cf5638a21e3"}, + {file = "coverage-7.8.2-cp313-cp313-win_arm64.whl", hash = "sha256:9fe449ee461a3b0c7105690419d0b0aba1232f4ff6d120a9e241e58a556733f7"}, + {file = "coverage-7.8.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8369a7c8ef66bded2b6484053749ff220dbf83cba84f3398c84c51a6f748a008"}, + {file = "coverage-7.8.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:159b81df53a5fcbc7d45dae3adad554fdbde9829a994e15227b3f9d816d00b36"}, + {file = "coverage-7.8.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6fcbbd35a96192d042c691c9e0c49ef54bd7ed865846a3c9d624c30bb67ce46"}, + {file = "coverage-7.8.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05364b9cc82f138cc86128dc4e2e1251c2981a2218bfcd556fe6b0fbaa3501be"}, + {file = "coverage-7.8.2-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46d532db4e5ff3979ce47d18e2fe8ecad283eeb7367726da0e5ef88e4fe64740"}, + {file = "coverage-7.8.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4000a31c34932e7e4fa0381a3d6deb43dc0c8f458e3e7ea6502e6238e10be625"}, + {file = "coverage-7.8.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:43ff5033d657cd51f83015c3b7a443287250dc14e69910577c3e03bd2e06f27b"}, + {file = "coverage-7.8.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94316e13f0981cbbba132c1f9f365cac1d26716aaac130866ca812006f662199"}, + {file = "coverage-7.8.2-cp313-cp313t-win32.whl", hash = "sha256:3f5673888d3676d0a745c3d0e16da338c5eea300cb1f4ada9c872981265e76d8"}, + {file = "coverage-7.8.2-cp313-cp313t-win_amd64.whl", hash = "sha256:2c08b05ee8d7861e45dc5a2cc4195c8c66dca5ac613144eb6ebeaff2d502e73d"}, + {file = "coverage-7.8.2-cp313-cp313t-win_arm64.whl", hash = "sha256:1e1448bb72b387755e1ff3ef1268a06617afd94188164960dba8d0245a46004b"}, + {file = "coverage-7.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:496948261eaac5ac9cf43f5d0a9f6eb7a6d4cb3bedb2c5d294138142f5c18f2a"}, + {file = "coverage-7.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eacd2de0d30871eff893bab0b67840a96445edcb3c8fd915e6b11ac4b2f3fa6d"}, + {file = "coverage-7.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b039ffddc99ad65d5078ef300e0c7eed08c270dc26570440e3ef18beb816c1ca"}, + {file = "coverage-7.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e49824808d4375ede9dd84e9961a59c47f9113039f1a525e6be170aa4f5c34d"}, + {file = "coverage-7.8.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b069938961dfad881dc2f8d02b47645cd2f455d3809ba92a8a687bf513839787"}, + {file = "coverage-7.8.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:de77c3ba8bb686d1c411e78ee1b97e6e0b963fb98b1637658dd9ad2c875cf9d7"}, + {file = "coverage-7.8.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1676628065a498943bd3f64f099bb573e08cf1bc6088bbe33cf4424e0876f4b3"}, + {file = "coverage-7.8.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:8e1a26e7e50076e35f7afafde570ca2b4d7900a491174ca357d29dece5aacee7"}, + {file = "coverage-7.8.2-cp39-cp39-win32.whl", hash = "sha256:6782a12bf76fa61ad9350d5a6ef5f3f020b57f5e6305cbc663803f2ebd0f270a"}, + {file = "coverage-7.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:1efa4166ba75ccefd647f2d78b64f53f14fb82622bc94c5a5cb0a622f50f1c9e"}, + {file = "coverage-7.8.2-pp39.pp310.pp311-none-any.whl", hash = "sha256:ec455eedf3ba0bbdf8f5a570012617eb305c63cb9f03428d39bf544cb2b94837"}, + {file = "coverage-7.8.2-py3-none-any.whl", hash = "sha256:726f32ee3713f7359696331a18daf0c3b3a70bb0ae71141b9d3c52be7c595e32"}, + {file = "coverage-7.8.2.tar.gz", hash = "sha256:a886d531373a1f6ff9fad2a2ba4a045b68467b779ae729ee0b3b10ac20033b27"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] + [[package]] name = "dateparser" version = "1.2.1" @@ -2127,6 +2210,44 @@ tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "1.0.0" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "pytest_asyncio-1.0.0-py3-none-any.whl", hash = "sha256:4f024da9f1ef945e680dc68610b52550e36590a67fd31bb3b4943979a1f90ef3"}, + {file = "pytest_asyncio-1.0.0.tar.gz", hash = "sha256:d15463d13f4456e1ead2594520216b225a16f781e144f8fdf6c5bb4667c48b3f"}, +] + +[package.dependencies] +pytest = ">=8.2,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + +[[package]] +name = "pytest-cov" +version = "6.1.1" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "pytest_cov-6.1.1-py3-none-any.whl", hash = "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde"}, + {file = "pytest_cov-6.1.1.tar.gz", hash = "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a"}, +] + +[package.dependencies] +coverage = {version = ">=7.5", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + [[package]] name = "python-cowsay" version = "1.2.0" @@ -3217,4 +3338,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "249a51a5424f3949d819da69d1947371d9fb44ccf4775e312d577105669273e4" +content-hash = "37d77db3e5032bfcaf5ae7e0734d2006db4b3275dc3da5448b93b5a4ba0bac5f" diff --git a/pyproject.toml b/pyproject.toml index 11a23db..4c4148b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ prospector = "^1.10.3" pytest = "^8.2.1" ipdb = "^0.13.13" psutil = "^7.0.0" +pytest-cov = "^6.1.1" +pytest-asyncio = "^1.0.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/src/bigmeow/settings.py b/src/bigmeow/settings.py index eabfcb7..3209ed3 100644 --- a/src/bigmeow/settings.py +++ b/src/bigmeow/settings.py @@ -32,6 +32,7 @@ DISCORD_USER = int(environ["DISCORD_USER"]) ECHO_WEBHOOK = "/webhook/echo" +ECHO_TOKEN = environ["ECHO_TOKEN"] TASK_DEFAULT_STORE = "default" TASK_DEFAULT_EXECUTOR = "default" diff --git a/src/bigmeow/web.py b/src/bigmeow/web.py index 21d9a73..bfdfceb 100644 --- a/src/bigmeow/web.py +++ b/src/bigmeow/web.py @@ -144,12 +144,16 @@ async def scheduled(request: Request, logger: BoundLogger = Depends(Logger())): @app.post(settings.ECHO_WEBHOOK, include_in_schema=False) -async def chat_post( +async def echo_message( request: Request, + x_echo_token: Annotated[str, Header()], x_channel: Annotated[str, Header()], x_destination: Annotated[str, Header()], logger: BoundLogger = Depends(Logger()), ) -> None: + if not settings.ECHO_TOKEN == x_echo_token: + raise Exception("Bad token") + # FIXME need auth text = (await request.body()).decode() @@ -181,4 +185,4 @@ async def chat_post( ) case _: - raise Exception("Invalid channel") + raise Exception("Invalid echo channel") From 09f9aa732d2057d415684bea43d3e6de70e30793 Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Thu, 19 Jun 2025 15:55:30 +0800 Subject: [PATCH 8/9] Add workflow support to bigmeow --- .github/workflows/image-builder-dev.yml | 20 +++++++++++++ .gitignore | 40 +++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .github/workflows/image-builder-dev.yml create mode 100644 .gitignore diff --git a/.github/workflows/image-builder-dev.yml b/.github/workflows/image-builder-dev.yml new file mode 100644 index 0000000..dead1a4 --- /dev/null +++ b/.github/workflows/image-builder-dev.yml @@ -0,0 +1,20 @@ +name: image builder dev +run-name: Building new docker containers +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: set lower case owner name + run: echo "OWNER_LC=${OWNER,,}" >> ${GITHUB_ENV} + env: + OWNER: "${{ github.repository_owner }}" + - name: set repository branch name + run: echo "BRANCH_NAME=${BRANCH_NAME,,}" >> ${GITHUB_ENV} + env: + BRANCH_NAME: "${{ github.head_ref || github.ref_name }}" + - uses: actions/checkout@v4 + - run: sudo apt-get install -y podman + - run: echo ${{ secrets.GITHUB_TOKEN }} | podman login ghcr.io -u ${{ github.repository_owner }} --password-stdin + - run: podman build --tag "ghcr.io/$OWNER_LC/bigmeow_bot:$BRANCH_NAME" . + - run: podman push "ghcr.io/$OWNER_LC/bigmeow_bot:$BRANCH_NAME" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7e215f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# Virtual environments +.venv +.env +.envrc + +# Node + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + From 3fe07bd9b74737b74a7b68b7f47c19997f337bf1 Mon Sep 17 00:00:00 2001 From: Jeffrey04 Date: Thu, 19 Jun 2025 16:33:19 +0800 Subject: [PATCH 9/9] updated github action to avoid building main/master tag --- .github/workflows/image-builder-dev.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/image-builder-dev.yml b/.github/workflows/image-builder-dev.yml index dead1a4..54efce6 100644 --- a/.github/workflows/image-builder-dev.yml +++ b/.github/workflows/image-builder-dev.yml @@ -6,15 +6,21 @@ jobs: runs-on: ubuntu-latest steps: - name: set lower case owner name - run: echo "OWNER_LC=${OWNER,,}" >> ${GITHUB_ENV} env: OWNER: "${{ github.repository_owner }}" - - name: set repository branch name - run: echo "BRANCH_NAME=${BRANCH_NAME,,}" >> ${GITHUB_ENV} + run: echo "OWNER_LC=${OWNER,,}" >> ${GITHUB_ENV} + - name: set image tag name + shell: bash env: BRANCH_NAME: "${{ github.head_ref || github.ref_name }}" + run: | + if [[ "${BRANCH_NAME,,}" == "main" || "${BRANCH_NAME,,}" == "master" ]]; then + echo "IMAGE_TAG=latest" >> ${GITHUB_ENV} + else + echo "IMAGE_TAG=${BRANCH_NAME,,}" >> ${GITHUB_ENV} + fi - uses: actions/checkout@v4 - run: sudo apt-get install -y podman - run: echo ${{ secrets.GITHUB_TOKEN }} | podman login ghcr.io -u ${{ github.repository_owner }} --password-stdin - - run: podman build --tag "ghcr.io/$OWNER_LC/bigmeow_bot:$BRANCH_NAME" . - - run: podman push "ghcr.io/$OWNER_LC/bigmeow_bot:$BRANCH_NAME" + - run: podman build --tag "ghcr.io/$OWNER_LC/bigmeow_bot:$IMAGE_TAG" . + - run: podman push "ghcr.io/$OWNER_LC/bigmeow_bot:$IMAGE_TAG"