From a6fce057c70943ea948aed4c61b0faf66b9c1ae4 Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Sat, 10 Feb 2018 16:30:13 +0100 Subject: [PATCH 01/11] Pass update mentions to the event handler --- pyrogram/client/client.py | 53 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/pyrogram/client/client.py b/pyrogram/client/client.py index 990ccede..467edeef 100644 --- a/pyrogram/client/client.py +++ b/pyrogram/client/client.py @@ -274,36 +274,30 @@ class Client: self.fetch_peers(update.chats) for i in update.updates: - self.event_queue.put(i) - elif isinstance(update, types.UpdateShortMessage): - if update.user_id not in self.peers_by_id: - diff = self.send( - functions.updates.GetDifference( - pts=update.pts - 1, - date=update.date, - qts=-1 - ) + self.event_queue.put((i, update.users, update.chats)) + elif isinstance(update, (types.UpdateShortMessage, types.UpdateShortChatMessage)): + diff = self.send( + functions.updates.GetDifference( + pts=update.pts - update.pts_count, + date=update.date, + qts=-1 ) + ) - self.fetch_peers(diff.users) + self.fetch_peers(diff.users) + self.fetch_peers(diff.chats) - self.event_queue.put(update) - elif isinstance(update, types.UpdateShortChatMessage): - if update.chat_id not in self.peers_by_id: - diff = self.send( - functions.updates.GetDifference( - pts=update.pts - 1, - date=update.date, - qts=-1 - ) - ) - - self.fetch_peers(diff.users) - self.fetch_peers(diff.chats) - - self.event_queue.put(update) + self.event_queue.put(( + types.UpdateNewMessage( + message=diff.new_messages[0], + pts=update.pts, + pts_count=update.pts_count + ), + diff.users, + diff.chats + )) elif isinstance(update, types.UpdateShort): - self.event_queue.put(update.update) + self.event_queue.put((update.update, [], [])) except Exception as e: log.error(e, exc_info=True) @@ -321,7 +315,12 @@ class Client: try: if self.event_handler: - self.event_handler(self, event) + self.event_handler( + self, + event[0], + {i.id: i for i in event[1]}, + {i.id: i for i in event[2]} + ) except Exception as e: log.error(e, exc_info=True) From 85b573d1e578f25215d3bd2aab034d6e628beadf Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Sat, 10 Feb 2018 18:28:11 +0100 Subject: [PATCH 02/11] Set NET_WORKERS to 1 --- pyrogram/session/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrogram/session/session.py b/pyrogram/session/session.py index e3e236b6..f47cbc24 100644 --- a/pyrogram/session/session.py +++ b/pyrogram/session/session.py @@ -60,7 +60,7 @@ class Session: ) INITIAL_SALT = 0x616e67656c696361 - NET_WORKERS = 2 + NET_WORKERS = 1 WAIT_TIMEOUT = 10 MAX_RETRIES = 5 ACKS_THRESHOLD = 8 From dc94386507c047001c2060cb3606bcd44e8c7b1d Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Sun, 11 Feb 2018 13:19:52 +0100 Subject: [PATCH 03/11] Handle pts --- pyrogram/client/client.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pyrogram/client/client.py b/pyrogram/client/client.py index 467edeef..4b0dd00a 100644 --- a/pyrogram/client/client.py +++ b/pyrogram/client/client.py @@ -133,6 +133,8 @@ class Client: self.peers_by_id = {} self.peers_by_username = {} + self.channels_pts = {} + self.markdown = Markdown(self.peers_by_id) self.html = HTML(self.peers_by_id) @@ -274,6 +276,28 @@ class Client: self.fetch_peers(update.chats) for i in update.updates: + channel_id = getattr( + getattr( + getattr( + i, "message", None + ), "to_id", None + ), "channel_id", None + ) or getattr(i, "channel_id", None) + + pts = getattr(i, "pts", None) + + if channel_id and pts: + if channel_id not in self.channels_pts: + self.channels_pts[channel_id] = [] + + if pts in self.channels_pts[channel_id]: + continue + + self.channels_pts[channel_id].append(pts) + + if len(self.channels_pts[channel_id]) > 50: + self.channels_pts[channel_id] = self.channels_pts[channel_id][25:] + self.event_queue.put((i, update.users, update.chats)) elif isinstance(update, (types.UpdateShortMessage, types.UpdateShortChatMessage)): diff = self.send( From e458e25786ff3547457bf3642a65c97ebaeb1bbd Mon Sep 17 00:00:00 2001 From: 1pyxa1 <1pyxa1@gmail.com> Date: Sun, 11 Feb 2018 17:05:42 +0300 Subject: [PATCH 04/11] Add USERNAME_OCCUPIED error (#20) --- compiler/error/source/400_BAD_REQUEST.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/error/source/400_BAD_REQUEST.tsv b/compiler/error/source/400_BAD_REQUEST.tsv index 762424b9..ac1989b8 100644 --- a/compiler/error/source/400_BAD_REQUEST.tsv +++ b/compiler/error/source/400_BAD_REQUEST.tsv @@ -46,3 +46,4 @@ CHAT_ADMIN_REQUIRED The method requires admin privileges PHONE_NUMBER_BANNED The phone number is banned ABOUT_TOO_LONG The about text is too long MULTI_MEDIA_TOO_LONG The album contains more than 10 items +USERNAME_OCCUPIED The username is already in use From 8b7f7440b20c701bebc66bce6681ec501929cbd3 Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Mon, 12 Feb 2018 16:39:57 +0100 Subject: [PATCH 05/11] Remove reference --- pyrogram/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrogram/client/client.py b/pyrogram/client/client.py index 4b0dd00a..099a0cb6 100644 --- a/pyrogram/client/client.py +++ b/pyrogram/client/client.py @@ -379,7 +379,7 @@ class Client: self.event_handler = callback def send(self, data: Object): - """Use this method to send :ref:`Raw Function ` queries. + """Use this method to send Raw Function queries. This method makes possible to manually call every single Telegram API method in a low-level manner. Available functions are listed in the :obj:`pyrogram.api.functions` package and may accept compound From df89669e04a61f1d2647f57a48b20036a8862543 Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Tue, 13 Feb 2018 12:08:10 +0100 Subject: [PATCH 06/11] Revert back to "set_update_handler" name --- pyrogram/client/client.py | 86 ++++++++++++++++++------------------- pyrogram/session/session.py | 2 +- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/pyrogram/client/client.py b/pyrogram/client/client.py index 099a0cb6..b3cc7630 100644 --- a/pyrogram/client/client.py +++ b/pyrogram/client/client.py @@ -97,12 +97,12 @@ class Client: be an empty string: "" workers (:obj:`int`, optional): - Thread pool size for handling incoming events (updates). Defaults to 4. + Thread pool size for handling incoming updates. Defaults to 4. """ INVITE_LINK_RE = re.compile(r"^(?:https?://)?t\.me/joinchat/(.+)$") DIALOGS_AT_ONCE = 100 - UPDATE_WORKERS = 2 + UPDATES_WORKERS = 2 def __init__(self, session_name: str, @@ -144,9 +144,9 @@ class Client: self.is_idle = Event() - self.event_handler = None + self.updates_queue = Queue() self.update_queue = Queue() - self.event_queue = Queue() + self.update_handler = None def start(self): """Use this method to start the Client after creating it. @@ -179,11 +179,11 @@ class Client: self.rnd_id = self.session.msg_id self.get_dialogs() - for i in range(self.UPDATE_WORKERS): - Thread(target=self.update_worker, name="UpdateWorker#{}".format(i + 1)).start() + for i in range(self.UPDATES_WORKERS): + Thread(target=self.updates_worker, name="UpdatesWorker#{}".format(i + 1)).start() for i in range(self.workers): - Thread(target=self.event_worker, name="EventWorker#{}".format(i + 1)).start() + Thread(target=self.update_worker, name="UpdateWorker#{}".format(i + 1)).start() mimetypes.init() @@ -193,11 +193,11 @@ class Client: """ self.session.stop() - for i in range(self.UPDATE_WORKERS): - self.update_queue.put(None) + for _ in range(self.UPDATES_WORKERS): + self.updates_queue.put(None) - for i in range(self.workers): - self.event_queue.put(None) + for _ in range(self.workers): + self.update_queue.put(None) def fetch_peers(self, entities: list): for entity in entities: @@ -260,31 +260,31 @@ class Client: if username is not None: self.peers_by_username[username] = input_peer - def update_worker(self): + def updates_worker(self): name = threading.current_thread().name log.debug("{} started".format(name)) while True: - update = self.update_queue.get() + updates = self.updates_queue.get() - if update is None: + if updates is None: break try: - if isinstance(update, (types.Update, types.UpdatesCombined)): - self.fetch_peers(update.users) - self.fetch_peers(update.chats) + if isinstance(updates, (types.Update, types.UpdatesCombined)): + self.fetch_peers(updates.users) + self.fetch_peers(updates.chats) - for i in update.updates: + for update in updates.updates: channel_id = getattr( getattr( getattr( - i, "message", None + update, "message", None ), "to_id", None ), "channel_id", None - ) or getattr(i, "channel_id", None) + ) or getattr(update, "channel_id", None) - pts = getattr(i, "pts", None) + pts = getattr(update, "pts", None) if channel_id and pts: if channel_id not in self.channels_pts: @@ -298,12 +298,12 @@ class Client: if len(self.channels_pts[channel_id]) > 50: self.channels_pts[channel_id] = self.channels_pts[channel_id][25:] - self.event_queue.put((i, update.users, update.chats)) - elif isinstance(update, (types.UpdateShortMessage, types.UpdateShortChatMessage)): + self.update_queue.put((update, updates.users, updates.chats)) + elif isinstance(updates, (types.UpdateShortMessage, types.UpdateShortChatMessage)): diff = self.send( functions.updates.GetDifference( - pts=update.pts - update.pts_count, - date=update.date, + pts=updates.pts - updates.pts_count, + date=updates.date, qts=-1 ) ) @@ -311,39 +311,39 @@ class Client: self.fetch_peers(diff.users) self.fetch_peers(diff.chats) - self.event_queue.put(( + self.update_queue.put(( types.UpdateNewMessage( message=diff.new_messages[0], - pts=update.pts, - pts_count=update.pts_count + pts=updates.pts, + pts_count=updates.pts_count ), diff.users, diff.chats )) - elif isinstance(update, types.UpdateShort): - self.event_queue.put((update.update, [], [])) + elif isinstance(updates, types.UpdateShort): + self.update_queue.put((updates.update, [], [])) except Exception as e: log.error(e, exc_info=True) log.debug("{} stopped".format(name)) - def event_worker(self): + def update_worker(self): name = threading.current_thread().name log.debug("{} started".format(name)) while True: - event = self.event_queue.get() + update = self.update_queue.get() - if event is None: + if update is None: break try: - if self.event_handler: - self.event_handler( + if self.update_handler: + self.update_handler( self, - event[0], - {i.id: i for i in event[1]}, - {i.id: i for i in event[2]} + update[0], + {i.id: i for i in update[1]}, + {i.id: i for i in update[2]} ) except Exception as e: log.error(e, exc_info=True) @@ -368,15 +368,15 @@ class Client: self.is_idle.wait() - def set_event_handler(self, callback: callable): - """Use this method to set the event handler. + def set_update_handler(self, callback: callable): + """Use this method to set the update handler. Args: callback (:obj:`callable`): - A function that takes ``client, event`` as positional arguments. - It will be called when a new event is generated on your account. + A function that takes "client, update, users, chats" as positional arguments. + It will be called when a new update is received from the server. """ - self.event_handler = callback + self.update_handler = callback def send(self, data: Object): """Use this method to send Raw Function queries. diff --git a/pyrogram/session/session.py b/pyrogram/session/session.py index f47cbc24..8e56911f 100644 --- a/pyrogram/session/session.py +++ b/pyrogram/session/session.py @@ -270,7 +270,7 @@ class Session: msg_id = msg.body.msg_id else: if self.client is not None: - self.client.update_queue.put(msg.body) + self.client.updates_queue.put(msg.body) if msg_id in self.results: self.results[msg_id].value = getattr(msg.body, "result", msg.body) From c6d5fb4178f53987662775c8a6e8eaae529c5dfa Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Tue, 13 Feb 2018 13:56:09 +0100 Subject: [PATCH 07/11] Add much more detailed set_update_handler docstring --- pyrogram/client/client.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/pyrogram/client/client.py b/pyrogram/client/client.py index b3cc7630..1f1b2ce1 100644 --- a/pyrogram/client/client.py +++ b/pyrogram/client/client.py @@ -371,10 +371,41 @@ class Client: def set_update_handler(self, callback: callable): """Use this method to set the update handler. + You must call this method *before* you *start()* the Client. + Args: callback (:obj:`callable`): - A function that takes "client, update, users, chats" as positional arguments. - It will be called when a new update is received from the server. + A function that will be called when a new update is received from the server. It takes + :obj:`(client, update, users, chats)` as positional arguments (Look at the section below for + a detailed description). + + Other Parameters: + client (:obj:`pyrogram.Client`): + The Client itself, useful when you want to call other API methods inside the update handler. + + update (:obj:`types.Update `): + The received update, which can be one of the many single Updates listed in the *updates* + field you see in the :obj:`types.Update ` type. + + users (:obj:`dict`): + Dictionary of all :obj:`types.User ` mentioned in the update. + You can access extra info about the user (such as *first_name*, *last_name*, etc...) by using + the IDs you find in the *update* argument (e.g.: *users[1768841572]*). + + chats (:obj:`dict`): + Dictionary of all :obj:`types.Chat ` and + :obj:`types.Channel ` mentioned in the update. + You can access extra info about the chat (such as *title*, *participants_count*, etc...) + by using the IDs you find in the *update* argument (e.g.: *chats[1701277281]*). + + Note: + The following Empty or Forbidden types may exist inside the *users* and *chats* dictionaries. + They mean you have been blocked by the user or banned from the group/channel. + + - :obj:`types.UserEmpty ` + - :obj:`types.ChatEmpty ` + - :obj:`types.ChatForbidden ` + - :obj:`types.ChannelForbidden ` """ self.update_handler = callback From 560991498df03f8c5c094751b1cc3a04ea2cc250 Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Tue, 13 Feb 2018 13:59:29 +0100 Subject: [PATCH 08/11] Add MsgId lock --- pyrogram/session/internals/msg_id.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyrogram/session/internals/msg_id.py b/pyrogram/session/internals/msg_id.py index 583e0320..cf8c0402 100644 --- a/pyrogram/session/internals/msg_id.py +++ b/pyrogram/session/internals/msg_id.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with Pyrogram. If not, see . +from threading import Lock from time import time @@ -24,11 +25,13 @@ class MsgId: self.delta_time = delta_time self.last_time = 0 self.offset = 0 + self.lock = Lock() def __call__(self) -> int: - now = time() - self.offset = self.offset + 4 if now == self.last_time else 0 - msg_id = int((now + self.delta_time) * 2 ** 32) + self.offset - self.last_time = now + with self.lock: + now = time() + self.offset = self.offset + 4 if now == self.last_time else 0 + msg_id = int((now + self.delta_time) * 2 ** 32) + self.offset + self.last_time = now - return msg_id + return msg_id From aca6fa390e1d0f0554f9aa5894f8b8ae13b1c066 Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Tue, 13 Feb 2018 14:00:03 +0100 Subject: [PATCH 09/11] Add SeqNo lock --- pyrogram/session/internals/seq_no.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pyrogram/session/internals/seq_no.py b/pyrogram/session/internals/seq_no.py index 44a953c5..bef0d1a3 100644 --- a/pyrogram/session/internals/seq_no.py +++ b/pyrogram/session/internals/seq_no.py @@ -16,15 +16,19 @@ # You should have received a copy of the GNU Lesser General Public License # along with Pyrogram. If not, see . +from threading import Lock + class SeqNo: def __init__(self): self.content_related_messages_sent = 0 + self.lock = Lock() def __call__(self, is_content_related: bool) -> int: - seq_no = (self.content_related_messages_sent * 2) + (1 if is_content_related else 0) + with self.lock: + seq_no = (self.content_related_messages_sent * 2) + (1 if is_content_related else 0) - if is_content_related: - self.content_related_messages_sent += 1 + if is_content_related: + self.content_related_messages_sent += 1 - return seq_no + return seq_no From 78f2c9549c2cd592a4bf2286f738be5d293440be Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Tue, 13 Feb 2018 14:28:45 +0100 Subject: [PATCH 10/11] Modify resolve_peer to accommodate Peer* types --- pyrogram/client/client.py | 44 +++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/pyrogram/client/client.py b/pyrogram/client/client.py index 1f1b2ce1..d87b2042 100644 --- a/pyrogram/client/client.py +++ b/pyrogram/client/client.py @@ -703,25 +703,33 @@ class Client: return input_peer - def resolve_peer(self, peer_id: int or str): - if peer_id in ("self", "me"): - return InputPeerSelf() - else: - if type(peer_id) is str: - peer_id = peer_id.lower().strip("@") + def resolve_peer(self, peer_id: int or str or types.PeerUser or types.PeerChat or types.PeerChannel): + if type(peer_id) is str: + if peer_id in ("self", "me"): + return InputPeerSelf() - try: - return self.peers_by_username[peer_id] - except KeyError: - return self.resolve_username(peer_id) - else: - try: - return self.peers_by_id[peer_id] - except KeyError: - try: - return self.peers_by_id[int("-100" + str(peer_id))] - except KeyError: - raise PeerIdInvalid + peer_id = peer_id.lower().strip("@") + + try: + return self.peers_by_username[peer_id] + except KeyError: + return self.resolve_username(peer_id) + + if type(peer_id) is not int: + if isinstance(peer_id, types.PeerUser): + peer_id = peer_id.user_id + elif isinstance(peer_id, types.PeerChat): + peer_id = peer_id.chat_id + elif isinstance(peer_id, types.PeerChannel): + peer_id = int("-100" + str(peer_id.channel_id)) + + try: + return self.peers_by_id[peer_id] + except KeyError: + try: + return self.peers_by_id[int("-100" + str(peer_id))] + except KeyError: + raise PeerIdInvalid def get_me(self): """A simple method for testing the user authorization. Requires no parameters. From 89200f4697517ff3edfec27b5083c099cc8de49e Mon Sep 17 00:00:00 2001 From: Dan <14043624+delivrance@users.noreply.github.com> Date: Tue, 13 Feb 2018 16:24:04 +0100 Subject: [PATCH 11/11] Add resolve_peer docstrings --- pyrogram/client/client.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/pyrogram/client/client.py b/pyrogram/client/client.py index d87b2042..511a3e45 100644 --- a/pyrogram/client/client.py +++ b/pyrogram/client/client.py @@ -383,7 +383,7 @@ class Client: client (:obj:`pyrogram.Client`): The Client itself, useful when you want to call other API methods inside the update handler. - update (:obj:`types.Update `): + update (:obj:`Update`): The received update, which can be one of the many single Updates listed in the *updates* field you see in the :obj:`types.Update ` type. @@ -703,7 +703,26 @@ class Client: return input_peer - def resolve_peer(self, peer_id: int or str or types.PeerUser or types.PeerChat or types.PeerChannel): + def resolve_peer(self, peer_id: int or str): + """Use this method to get the *InputPeer* of a known *peer_id*. + + It is intended to be used when working with Raw Functions (i.e: a Telegram API method you wish to use which is + not available yet in the Client class as an easy-to-use method). + + Args: + peer_id (:obj:`int` | :obj:`str` | :obj:`Peer`): + The Peer ID you want to extract the InputPeer from. Can be one of these types: :obj:`int` (direct ID), + :obj:`str` (@username), :obj:`PeerUser `, + :obj:`PeerChat `, :obj:`PeerChannel ` + + Returns: + :obj:`InputPeerUser ` or + :obj:`InputPeerChat ` or + :obj:`InputPeerChannel ` depending on the *peer_id*. + + Raises: + :class:`pyrogram.Error` + """ if type(peer_id) is str: if peer_id in ("self", "me"): return InputPeerSelf()