advatroniks/Telegram-Scrapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add fix

Browse Source
This commit is contained in:
harold 2025-03-25 08:49:59 +05:00
parent ef3e20f8f7
commit fcc0c81d7c
9 changed files with 109 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
telegram-application/docker-compose.yml
View File

@ -7,8 +7,8 @@ services:
restart: unless-stopped
env_file:
- .env_prod
networks:
- app_network
# networks:
# - app_network
depends_on:
postgres:
condition: service_started
@ -30,8 +30,8 @@ services:
volumes:
- ./docker/postgres/data:/var/lib/postgresql/data
- ./sql_scripts/create_table.sql:/docker-entrypoint-initdb.d/create_table.sql
networks:
- app_network
# networks:
# - app_network
redis:
image: redis:latest
@ -39,8 +39,8 @@ services:
restart: unless-stopped
ports:
- "6379:6379"
networks:
- app_network
# networks:
# - app_network
rabbitmq:
image: "rabbitmq:3-management"
@ -57,9 +57,9 @@ services:
interval: 10s
timeout: 5s
retries: 5
networks:
- app_network
# networks:
# - app_network
networks:
app_network:
external: true
#networks:
# app_network:
# external: true

2
telegram-application/src/core/tg_service/constants.py
View File

@ -1 +1 @@
MESSAGE_CHANG_SIZE: int = 20
MESSAGE_CHANG_SIZE: int = 30

17
telegram-application/src/core/tg_service/messages_handler.py
View File

@ -1,3 +1,5 @@
import asyncio
from pyrogram import Client
from pyrogram.types import Message
@ -9,15 +11,18 @@ from src.core.tg_service import utils as api_tg_utils
DATA: dict[int, list[MessageFromChatSchema]] = dict()
lock = asyncio.Lock()
async def message_listener(client: Client, message: Message):
print("received message")
if api_tg_utils.check_message_condition(message):
await check_user_exists(
user_pyrogram=message.from_user,
)
await check_chat_exists(
chat_pyrogram=message.chat,
)
async with lock:
await check_user_exists(
user_pyrogram=message.from_user,
)
await check_chat_exists(
chat_pyrogram=message.chat,
)
message_schema = MessageFromChatSchema(
id=message.id,

2
telegram-application/src/core/tg_service/notify_sender.py
View File

@ -18,4 +18,6 @@ async def send_to_tg_from_bot(text: str):
async with AsyncClient() as client:
response = await client.post(url, json=payload)
print(response)
if response.status_code != 200:
print(response.text)
return response

6
telegram-application/src/core/tg_service/notify_success.py
View File

@ -17,6 +17,6 @@ async def notify_for_success(
chat=chat,
)
# await send_to_tg_from_bot(
# text=message
# )
await send_to_tg_from_bot(
text=message
)

93
telegram-application/src/core/tg_service/utils.py
View File

@ -21,22 +21,85 @@ def create_and_format_message(
chat: TgChat,
user_model: User,
) -> str:
if user_model.username:
user_link = f"[{user_model.username}](tg://user?id={user_model.id})"
else:
user_link = f"ID: {user_model.id}"
def escape_markdown_v2(text: str) -> str:
escape_chars = '_*[]()~`>#+-=|{}.!'
return ''.join('\\' + char if char in escape_chars else char for char in text)
messages_text = "\n".join(
f"{msg.message_time.isoformat()}: {msg.text}\n" for msg in messages
# Формирование информации о пользователе
username = escape_markdown_v2(user_model.username) if user_model.username else f"ID: {user_model.id}"
user_link = f"[{username}](tg://user?id={user_model.id})" if user_model.username else f"ID: {user_model.id}"
# Формирование информации о чате
chat_title = escape_markdown_v2(chat.title)
chat_link = (
f"https://t.me/c/{str(chat.id)[4:]}"
if str(chat.id).startswith("-100")
else f"https://t.me/{chat.username}" if chat.username
else f"Chat ID: {chat.id}"
)
messages_text = '```dialog\n' + messages_text + '```'
chat_link = f"https://t.me/c/{str(chat.id)[4:]}" if str(chat.id).startswith(
"-100") else f"https://t.me/{chat.id}"
# Экранирование причины
reason_escaped = escape_markdown_v2(reason)
return f""" 🔥 *Найдена успешка!*
👤 *Пользователь:* {user_link}\n
🐩 *Чат*: [{chat.title}]({chat_link})
📌 *Причина:* {reason}
📝 *Диалог:* {messages_text}
"""
# Базовый заголовок сообщения
header = (
f"🔥 *Найдена успешка!*\n"
f"👤 *Пользователь:* {user_link}\n"
f"🐩 *Чат:* [{chat_title}]({chat_link})\n"
f"📌 *Причина:* {reason_escaped}\n\n"
f"📝 *Диалог:*\n"
)
# Расчет доступной длины для контента
MAX_LENGTH = 4096
header_length = len(header)
available_length = MAX_LENGTH - header_length
# Формирование блоков сообщений
message_blocks = []
current_length = 0
for msg in messages:
# Очистка и подготовка текста
clean_text = msg.text.replace('```', 'ʻʻʻ') # Заменяем опасные символы
escaped_text = escape_markdown_v2(clean_text)
sender_username = escape_markdown_v2(
msg.user_relationship.username or str(msg.user_id)
)
# Формирование блока сообщения
block = (
f"**{sender_username}:**\n"
f"```\n"
f"{escaped_text}\n"
f"```\n\n"
)
block_length = len(block)
# Проверка на превышение длины
if current_length + block_length > available_length:
remaining_space = available_length - current_length
if remaining_space > 20: # Минимальный значимый блок
truncated_text = escaped_text[:remaining_space - 20] + "..."
block = (
f"**Пользователь:**\n"
f"```\n"
f"{truncated_text}\n"
f"```\n\n"
f"_... сообщение обрезано ..._"
)
message_blocks.append(block)
break
message_blocks.append(block)
current_length += block_length
# Сборка финального сообщения
content = ''.join(message_blocks).strip()
full_message = header + content
# Финальная проверка длины
if len(full_message) > MAX_LENGTH:
full_message = full_message[:MAX_LENGTH - 17] + "\n```\n...\n```\n_... сообщение обрезано ..._"
return full_message

2
telegram-application/src/core/workers/crud.py
View File

@ -1,6 +1,7 @@
from uuid import UUID
from sqlalchemy import insert, select
from sqlalchemy.orm import joinedload
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.database import TgChat, TgMessage
@ -30,6 +31,7 @@ async def get_messages_by_slice_id(
) -> list[TgMessage]:
stmt = (
select(TgMessage)
.options(joinedload(TgMessage.user_relationship))
.where(
TgMessage.slice_id == slice_id
)

1
telegram-application/src/core/workers/rmq_worker_handler.py
View File

@ -17,6 +17,7 @@ async def create_success_record(
message: ResponseFromGeminiSchema,
session: Annotated[AsyncSession, Depends(db_helper.get_async_session)],
):
print(message.success)
await workers_crud.bulk_create_success_reasons(
success_schema=message,
session=session,

BIN
telegram-application/src/tg_account.session
View File

Binary file not shown.