Add API compiler

compiler/__init__.py

@@ -0,0 +1,17 @@
+# Pyrogram - Telegram MTProto API Client Library for Python
+# Copyright (C) 2017 Dan Tès <https://github.com/delivrance>
+#
+# This file is part of Pyrogram.
+#
+# Pyrogram is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pyrogram is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Pyrogram.  If not, see <http://www.gnu.org/licenses/>.

compiler/api/__init__.py

@@ -0,0 +1,17 @@
+# Pyrogram - Telegram MTProto API Client Library for Python
+# Copyright (C) 2017 Dan Tès <https://github.com/delivrance>
+#
+# This file is part of Pyrogram.
+#
+# Pyrogram is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pyrogram is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Pyrogram.  If not, see <http://www.gnu.org/licenses/>.

compiler/api/compiler.py

@@ -0,0 +1,313 @@
+# Pyrogram - Telegram MTProto API Client Library for Python
+# Copyright (C) 2017 Dan Tès <https://github.com/delivrance>
+#
+# This file is part of Pyrogram.
+#
+# Pyrogram is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pyrogram is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Pyrogram.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import shutil
+
+home = "compiler/api"
+dest = "pyrogram/api"
+notice_path = "NOTICE"
+
+core_types = ["int", "long", "int128", "int256", "double", "bytes", "string", "Bool"]
+
+
+# TODO: Compiler was written in a rush. variables/methods name and pretty much all the code is fuzzy, but it works
+# TODO: Some constructors have flags:# but not flags.\d+\?
+
+class Compiler:
+    def __init__(self):
+        self.section = "types"  # TL Schema starts with types
+        self.namespaces = {"types": set(), "functions": set()}
+        self.objects = {}
+        self.layer = None
+
+        self.schema = None
+
+        with open("{}/template/class.txt".format(home)) as f:
+            self.template = f.read()
+
+        with open(notice_path) as f:
+            notice = []
+
+            for line in f.readlines():
+                notice.append("# {}".format(line).strip())
+
+            self.notice = "\n".join(notice)
+
+    def read_schema(self):
+        """Read schema files"""
+        with open("{}/source/auth_key.tl".format(home)) as auth, \
+                open("{}/source/sys_msgs.tl".format(home)) as system, \
+                open("{}/source/main_api.tl".format(home)) as api:
+            self.schema = auth.read() + system.read() + api.read()
+
+    def parse_schema(self):
+        """Parse schema line by line"""
+        total = len(self.schema.splitlines())
+
+        for i, line in enumerate(self.schema.splitlines()):
+            # Check for section changer lines
+            section = re.match(r"---(\w+)---", line)
+            if section:
+                self.section = section.group(1)
+                continue
+
+            # Save the layer version
+            layer = re.match(r"//\sLAYER\s(\d+)", line)
+            if layer:
+                self.layer = layer.group(1)
+                continue
+
+            combinator = re.match(r"^([\w.]+)#([0-9a-f]+)\s(.*)=\s([\w<>.]+);$", line, re.MULTILINE)
+
+            if combinator:
+                name, id, args, type = combinator.groups()
+                namespace, name = name.split(".") if "." in name else ("", name)
+                args = re.findall(r"[^{](\w+):([\w?!.<>]+)", line)
+
+                print("Compiling APIs: [{}%]".format(round(i * 100 / total)), end="\r")
+
+                for i in args:
+                    if re.match(r"flags\.\d+\?", i[1]):
+                        has_flags = True
+                        break
+                else:
+                    has_flags = False
+
+                if name == "updates":
+                    name = "update"
+
+                for i, item in enumerate(args):
+                    # if item[0] in keyword.kwlist + dir(builtins) + ["self"]:
+                    if item[0] == "self":
+                        args[i] = ("is_{}".format(item[0]), item[1])
+
+                if namespace:
+                    self.namespaces[self.section].add(namespace)
+
+                self.compile(namespace, name, id, args, has_flags)
+
+                self.objects[id] = "{}.{}{}.{}".format(
+                    self.section,
+                    "{}.".format(namespace) if namespace else "",
+                    self.snek(name),
+                    self.caml(name)
+                )
+
+    def finish(self):
+        with open("{}/all.py".format(dest), "w") as f:
+            f.write(self.notice + "\n\n")
+            f.write("layer = {}\n\n".format(self.layer))
+            f.write("objects = {")
+
+            for k, v in self.objects.items():
+                v = v.split(".")
+                del v[-2]
+                v = ".".join(v)
+
+                f.write("\n    0x{}: \"{}\",".format(k.zfill(8), v))
+
+            f.write("\n    0xbc799737: \"core.BoolFalse\",")
+            f.write("\n    0x997275b5: \"core.BoolTrue\",")
+            f.write("\n    0x1cb5c415: \"core.Vector\",")
+            f.write("\n    0x73f1f8dc: \"core.MsgContainer\",")
+            f.write("\n    0xae500895: \"core.FutureSalts\",")
+            f.write("\n    0x0949d9dc: \"core.FutureSalt\",")
+            f.write("\n    0x3072cfa1: \"core.GzipPacked\",")
+            f.write("\n    0x5bb8e511: \"core.Message\"")
+
+            f.write("\n}\n")
+
+        for k, v in self.namespaces.items():
+            with open("{}/{}/__init__.py".format(dest, k), "a") as f:
+                f.write("from . import {}\n".format(", ".join([i for i in v])) if v else "")
+
+    @staticmethod
+    def sort_args(args):
+        """Put flags at the end"""
+        args = args.copy()
+        flags = [i for i in args if re.match(r"flags\.\d+\?", i[1])]
+
+        for i in flags:
+            args.remove(i)
+
+        return args + flags
+
+    def compile(self, namespace, name, id, args, has_flags):
+        path = "{}/{}/{}".format(dest, self.section, namespace)
+        os.makedirs(path, exist_ok=True)
+
+        init = "{}/__init__.py".format(path)
+
+        if not os.path.exists(init):
+            with open(init, "w") as f:
+                f.write(self.notice + "\n\n")
+
+        with open(init, "a") as f:
+            f.write("from .{} import {}\n".format(self.snek(name), self.caml(name)))
+
+        sorted_args = self.sort_args(args)
+
+        object_id = "0x{}".format(id)
+
+        arguments = ", " + ", ".join(
+            ["{}{}".format(
+                i[0],
+                "=None" if re.match(r"flags\.\d+\?", i[1]) else ""
+            ) for i in sorted_args]
+        ) if args else ""
+
+        fields = "\n        ".join(
+            ["self.{0} = {0}  # {1}".format(i[0], i[1]) for i in args]
+        ) if args else "pass"
+
+        if has_flags:
+            write_flags = []
+            for i in args:
+                flag = re.match(r"flags\.(\d+)\?", i[1])
+                if flag:
+                    write_flags.append("flags |= (1 << {}) if self.{} is not None else 0".format(flag.group(1), i[0]))
+
+            write_flags = "\n        ".join([
+                "flags = 0",
+                "\n        ".join(write_flags),
+                "b.write(Int(flags))"
+            ])
+        else:
+            write_flags = "# No flags"
+
+        read_flags = "flags = Int.read(b)" if has_flags else "# No flags"
+
+        write_types = read_types = ""
+
+        for arg_name, arg_type in args:
+            flag = re.findall(r"flags\.(\d+)\?([\w<>.]+)", arg_type)
+
+            if flag:
+                index, flag_type = flag[0]
+
+                if flag_type == "true":
+                    read_types += "\n        "
+                    read_types += "{} = True if flags & (1 << {}) else False".format(arg_name, index)
+                elif flag_type in core_types:
+                    write_types += "\n        "
+                    write_types += "if self.{} is not None:\n            ".format(arg_name)
+                    write_types += "b.write({}(self.{}))\n        ".format(flag_type.title(), arg_name)
+
+                    read_types += "\n        "
+                    read_types += "{} = {}.read(b) if flags & (1 << {}) else None".format(
+                        arg_name, flag_type.title(), index
+                    )
+                elif "vector" in flag_type.lower():
+                    sub_type = arg_type.split("<")[1][:-1]
+
+                    write_types += "\n        "
+                    write_types += "if self.{} is not None:\n            ".format(arg_name)
+                    write_types += "b.write(Vector(self.{}{}))\n        ".format(
+                        arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else ""
+                    )
+
+                    read_types += "\n        "
+                    read_types += "{} = Object.read(b{}) if flags & (1 << {}) else []\n        ".format(
+                        arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else "", index
+                    )
+                else:
+                    write_types += "\n        "
+                    write_types += "if self.{} is not None:\n            ".format(arg_name)
+                    write_types += "b.write(self.{}.write())\n        ".format(arg_name)
+
+                    read_types += "\n        "
+                    read_types += "{} = Object.read(b) if flags & (1 << {}) else None\n        ".format(
+                        arg_name, index
+                    )
+            else:
+                if arg_type in core_types:
+                    write_types += "\n        "
+                    write_types += "b.write({}(self.{}))\n        ".format(arg_type.title(), arg_name)
+
+                    read_types += "\n        "
+                    read_types += "{} = {}.read(b)\n        ".format(arg_name, arg_type.title())
+                elif "vector" in arg_type.lower():
+                    sub_type = arg_type.split("<")[1][:-1]
+
+                    write_types += "\n        "
+                    write_types += "b.write(Vector(self.{}{}))\n        ".format(
+                        arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else ""
+                    )
+
+                    read_types += "\n        "
+                    read_types += "{} = Object.read(b{})\n        ".format(
+                        arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else ""
+                    )
+                else:
+                    write_types += "\n        "
+                    write_types += "b.write(self.{}.write())\n        ".format(arg_name)
+
+                    read_types += "\n        "
+                    read_types += "{} = Object.read(b)\n        ".format(arg_name)
+
+        with open("{}/{}.py".format(path, self.snek(name)), "w") as f:
+            f.write(
+                self.template.format(
+                    notice=self.notice,
+                    class_name=self.caml(name),
+                    object_id=object_id,
+                    arguments=arguments,
+                    fields=fields,
+                    read_flags=read_flags,
+                    read_types=read_types,
+                    write_flags=write_flags,
+                    write_types=write_types,
+                    return_arguments=", ".join([i[0] for i in sorted_args])
+                )
+            )
+
+    @staticmethod
+    def snek(s):
+        # https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case
+        s = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", s)
+        return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s).lower()
+
+    @staticmethod
+    def caml(s):
+        s = Compiler.snek(s).split("_")
+        return "".join([str(i.title()) for i in s])
+
+    def start(self):
+        shutil.rmtree("{}/types".format(dest), ignore_errors=True)
+        shutil.rmtree("{}/functions".format(dest), ignore_errors=True)
+
+        self.read_schema()
+        self.parse_schema()
+        self.finish()
+
+        print()
+
+
+def start():
+    c = Compiler()
+    c.start()
+
+
+if "__main__" == __name__:
+    home = "."
+    dest = "../../pyrogram/api"
+    notice_path = "../../NOTICE"
+
+    start()

compiler/api/source/auth_key.tl

@@ -0,0 +1,37 @@
+///////////////////////////////
+/// Authorization key creation
+///////////////////////////////
+
+---types---
+
+resPQ#05162463 nonce:int128 server_nonce:int128 pq:bytes server_public_key_fingerprints:Vector<long> = ResPQ;
+
+p_q_inner_data#83c95aec pq:bytes p:bytes q:bytes nonce:int128 server_nonce:int128 new_nonce:int256 = P_Q_inner_data;
+p_q_inner_data_temp#3c6a84d4 pq:bytes p:bytes q:bytes nonce:int128 server_nonce:int128 new_nonce:int256 expires_in:int = P_Q_inner_data;
+
+bind_auth_key_inner#75a3f765 nonce:long temp_auth_key_id:long perm_auth_key_id:long temp_session_id:long expires_at:int = BindAuthKeyInner;
+
+server_DH_params_fail#79cb045d nonce:int128 server_nonce:int128 new_nonce_hash:int128 = Server_DH_Params;
+server_DH_params_ok#d0e8075c nonce:int128 server_nonce:int128 encrypted_answer:bytes = Server_DH_Params;
+
+server_DH_inner_data#b5890dba nonce:int128 server_nonce:int128 g:int dh_prime:bytes g_a:bytes server_time:int = Server_DH_inner_data;
+
+client_DH_inner_data#6643b654 nonce:int128 server_nonce:int128 retry_id:long g_b:bytes = Client_DH_Inner_Data;
+
+dh_gen_ok#3bcbf734 nonce:int128 server_nonce:int128 new_nonce_hash1:int128 = Set_client_DH_params_answer;
+dh_gen_retry#46dc1fb9 nonce:int128 server_nonce:int128 new_nonce_hash2:int128 = Set_client_DH_params_answer;
+dh_gen_fail#a69dae02 nonce:int128 server_nonce:int128 new_nonce_hash3:int128 = Set_client_DH_params_answer;
+
+destroy_auth_key_ok#f660e1d4 = DestroyAuthKeyRes;
+destroy_auth_key_none#0a9f2259 = DestroyAuthKeyRes;
+destroy_auth_key_fail#ea109b13 = DestroyAuthKeyRes;
+
+---functions---
+
+req_pq#60469778 nonce:int128 = ResPQ;
+
+req_DH_params#d712e4be nonce:int128 server_nonce:int128 p:bytes q:bytes public_key_fingerprint:long encrypted_data:bytes = Server_DH_Params;
+
+set_client_DH_params#f5045f1f nonce:int128 server_nonce:int128 encrypted_data:bytes = Set_client_DH_params_answer;
+
+destroy_auth_key#d1435160 = DestroyAuthKeyRes;

compiler/api/source/sys_msgs.tl

@@ -0,0 +1,62 @@
+///////////////////////////////
+////////////// System messages
+///////////////////////////////
+
+---types---
+
+msgs_ack#62d6b459 msg_ids:Vector<long> = MsgsAck;
+
+bad_msg_notification#a7eff811 bad_msg_id:long bad_msg_seqno:int error_code:int = BadMsgNotification;
+bad_server_salt#edab447b bad_msg_id:long bad_msg_seqno:int error_code:int new_server_salt:long = BadMsgNotification;
+
+msgs_state_req#da69fb52 msg_ids:Vector<long> = MsgsStateReq;
+msgs_state_info#04deb57d req_msg_id:long info:string = MsgsStateInfo;
+msgs_all_info#8cc0d131 msg_ids:Vector<long> info:string = MsgsAllInfo;
+
+msg_detailed_info#276d3ec6 msg_id:long answer_msg_id:long bytes:int status:int = MsgDetailedInfo;
+msg_new_detailed_info#809db6df answer_msg_id:long bytes:int status:int = MsgDetailedInfo;
+
+msg_resend_req#7d861a08 msg_ids:Vector<long> = MsgResendReq;
+msg_resend_ans_req#8610baeb msg_ids:Vector<long> = MsgResendReq;
+
+rpc_result#f35c6d01 req_msg_id:long result:Object = RpcResult;
+rpc_error#2144ca19 error_code:int error_message:string = RpcError;
+
+rpc_answer_unknown#5e2ad36e = RpcDropAnswer;
+rpc_answer_dropped_running#cd78e586 = RpcDropAnswer;
+rpc_answer_dropped#a43ad8b7 msg_id:long seq_no:int bytes:int = RpcDropAnswer;
+
+// future_salt#0949d9dc valid_since:int valid_until:int salt:long = FutureSalt;  // Parsed manually
+// future_salts#ae500895 req_msg_id:long now:int salts:vector<future_salt> = FutureSalts;  // Parsed manually
+
+pong#347773c5 msg_id:long ping_id:long = Pong;
+
+destroy_session_ok#e22045fc session_id:long = DestroySessionRes;
+destroy_session_none#62d350c9 session_id:long = DestroySessionRes;
+
+new_session_created#9ec20908 first_msg_id:long unique_id:long server_salt:long = NewSession;
+
+//// 0x5bb8e511 = hex(crc32(b"message msg_id:long seqno:int bytes:int body:Object = Message"))
+// message#5bb8e511 msg_id:long seqno:int bytes:int body:Object = Message;  // Parsed manually
+// msg_container#73f1f8dc messages:vector<message> = MessageContainer;  // Parsed manually
+// msg_copy#e06046b2 orig_message:Message = MessageCopy;  // Not used
+// gzip_packed#3072cfa1 packed_data:string = Object;  // Parsed manually
+
+//// 0xd433ad73 = hex(crc32(b"ipPort ipv4:int port:int = IpPort"))
+ipPort#d433ad73 ipv4:int port:int = IpPort;
+help.configSimple#d997c3c5 date:int expires:int dc_id:int ip_port_list:Vector<ipPort> = help.ConfigSimple;
+
+---functions---
+
+rpc_drop_answer#58e4a740 req_msg_id:long = RpcDropAnswer;
+
+get_future_salts#b921bd04 num:int = FutureSalts;
+
+ping#7abe77ec ping_id:long = Pong;
+ping_delay_disconnect#f3427b8c ping_id:long disconnect_delay:int = Pong;
+
+destroy_session#e7512126 session_id:long = DestroySessionRes;
+
+contest.saveDeveloperInfo#9a5f6e95 vk_id:int name:string phone_number:string age:int city:string = Bool;
+
+http_wait#9299359f max_delay:int wait_after:int max_wait:int = HttpWait;

compiler/api/template/class.txt

@@ -0,0 +1,26 @@
+{notice}
+
+from io import BytesIO
+
+from pyrogram.api.core import *
+
+
+class {class_name}(Object):
+    ID = {object_id}
+
+    def __init__(self{arguments}):
+        {fields}
+
+    @staticmethod
+    def read(b: BytesIO) -> "{class_name}":
+        {read_flags}
+        {read_types}
+        return {class_name}({return_arguments})
+
+    def write(self) -> bytes:
+        b = BytesIO()
+        b.write(Int(self.ID, False))
+
+        {write_flags}
+        {write_types}
+        return b.getvalue()