Rework API compiler

compiler/api/compiler.py

@@ -20,27 +20,68 @@ import os
 import re
 import shutil
 
-home = "compiler/api"
+HOME = "compiler/api"
-dest = "pyrogram/api"
+DESTINATION = "pyrogram/api"
 notice_path = "NOTICE"
+SECTION_RE = re.compile(r"---(\w+)---")
+LAYER_RE = re.compile(r"//\sLAYER\s(\d+)")
+COMBINATOR_RE = re.compile(r"^([\w.]+)#([0-9a-f]+)\s(?:.*)=\s([\w<>.]+);$", re.MULTILINE)
+ARGS_RE = re.compile("[^{](\w+):([\w?!.<>]+)")
+FLAGS_RE = re.compile(r"flags\.(\d+)\?")
+FLAGS_RE_2 = re.compile(r"flags\.(\d+)\?([\w<>.]+)")
 
 core_types = ["int", "long", "int128", "int256", "double", "bytes", "string", "Bool"]
 
 
-# TODO: Compiler was written in a rush. variables/methods name and pretty much all the code is fuzzy, but it works
+class Combinator:
-# TODO: Some constructors have flags:# but not flags.\d+\?
+    def __init__(self, section: str, namespace: str, name: str, id: str, args: list, has_flags: bool, return_type: str):
+        self.section = section
+        self.namespace = namespace
+        self.name = name
+        self.id = id
+        self.args = args
+        self.has_flags = has_flags
+        self.return_type = return_type
 
-class Compiler:
-    def __init__(self):
-        self.section = "types"  # TL Schema starts with types
-        self.namespaces = {"types": set(), "functions": set()}
-        self.objects = {}
-        self.layer = None
 
-        self.schema = None
+def snek(s: str):
+    # https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case
+    s = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", s)
+    return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s).lower()
 
-        with open("{}/template/class.txt".format(home)) as f:
+
-            self.template = f.read()
+def capit(s: str):
+    return "".join([i[0].upper() + i[1:] for i in s.split("_")])
+
+
+#
+# def caml(s: str):
+#     r = snek(s).split("_")
+#     return "".join([str(i.title()) for i in r])
+
+
+def sort_args(args):
+    """Put flags at the end"""
+    args = args.copy()
+    flags = [i for i in args if FLAGS_RE.match(i[1])]
+
+    for i in flags:
+        args.remove(i)
+
+    return args + flags
+
+
+def start():
+    shutil.rmtree("{}/types".format(DESTINATION), ignore_errors=True)
+    shutil.rmtree("{}/functions".format(DESTINATION), ignore_errors=True)
+
+    with open("{}/source/auth_key.tl".format(HOME)) as auth, \
+            open("{}/source/sys_msgs.tl".format(HOME)) as system, \
+            open("{}/source/main_api.tl".format(HOME)) as api:
+        schema = (auth.read() + system.read() + api.read()).splitlines()
+
+    with open("{}/template/class.txt".format(HOME)) as f:
+        template = f.read()
 
     with open(notice_path) as f:
         notice = []
@@ -48,135 +89,90 @@ class Compiler:
         for line in f.readlines():
             notice.append("# {}".format(line).strip())
 
-            self.notice = "\n".join(notice)
+        notice = "\n".join(notice)
 
-    def read_schema(self):
+    total = len(schema)
-        """Read schema files"""
+    section = None
-        with open("{}/source/auth_key.tl".format(home)) as auth, \
+    layer = None
-                open("{}/source/sys_msgs.tl".format(home)) as system, \
+    namespaces = {"types": set(), "functions": set()}
-                open("{}/source/main_api.tl".format(home)) as api:
+    combinators = []
-            self.schema = auth.read() + system.read() + api.read()
 
-    def parse_schema(self):
+    for i, line in enumerate(schema):
-        """Parse schema line by line"""
-        total = len(self.schema.splitlines())
-
-        for i, line in enumerate(self.schema.splitlines()):
         # Check for section changer lines
-            section = re.match(r"---(\w+)---", line)
+        s = SECTION_RE.match(line)
-            if section:
+        if s:
-                self.section = section.group(1)
+            section = s.group(1)
             continue
 
         # Save the layer version
-            layer = re.match(r"//\sLAYER\s(\d+)", line)
+        l = LAYER_RE.match(line)
-            if layer:
+        if l:
-                self.layer = layer.group(1)
+            layer = l.group(1)
             continue
 
-            combinator = re.match(r"^([\w.]+)#([0-9a-f]+)\s(.*)=\s([\w<>.]+);$", line, re.MULTILINE)
+        combinator = COMBINATOR_RE.match(line)
-
         if combinator:
-                name, id, args, type = combinator.groups()
+            name, id, return_type = combinator.groups()
             namespace, name = name.split(".") if "." in name else ("", name)
-                args = re.findall(r"[^{](\w+):([\w?!.<>]+)", line)
+            args = ARGS_RE.findall(line)
-
-                print("Compiling APIs: [{}%]".format(round(i * 100 / total)), end="\r")
 
+            # Check if combinator has flags
             for i in args:
-                    if re.match(r"flags\.\d+\?", i[1]):
+                if FLAGS_RE.match(i[1]):
                     has_flags = True
                     break
             else:
                 has_flags = False
 
+            # Fix file and folder name collision
             if name == "updates":
                 name = "update"
 
+            # Fix arg name being "self" (reserved keyword)
             for i, item in enumerate(args):
-                    # if item[0] in keyword.kwlist + dir(builtins) + ["self"]:
                 if item[0] == "self":
-                        args[i] = ("is_{}".format(item[0]), item[1])
+                    args[i] = ("is_self", item[1])
 
             if namespace:
-                    self.namespaces[self.section].add(namespace)
+                namespaces[section].add(namespace)
 
-                self.compile(namespace, name, id, args, has_flags, type)
+            combinators.append(
-
+                Combinator(
-                self.objects[id] = "{}.{}{}.{}".format(
+                    section,
-                    self.section,
+                    namespace,
-                    "{}.".format(namespace) if namespace else "",
+                    name,
-                    self.snek(name),
+                    "0x{}".format(id.zfill(8)),
-                    self.caml(name)
+                    args,
+                    has_flags,
+                    return_type
+                )
             )
 
-    def finish(self):
+    for c in combinators:  # type: Combinator
-        with open("{}/all.py".format(dest), "w") as f:
+        path = "{}/{}/{}".format(DESTINATION, c.section, c.namespace)
-            f.write(self.notice + "\n\n")
-            f.write("layer = {}\n\n".format(self.layer))
-            f.write("objects = {")
-
-            for k, v in self.objects.items():
-                v = v.split(".")
-                del v[-2]
-                v = ".".join(v)
-
-                f.write("\n    0x{}: \"{}\",".format(k.zfill(8), v))
-
-            f.write("\n    0xbc799737: \"core.BoolFalse\",")
-            f.write("\n    0x997275b5: \"core.BoolTrue\",")
-            f.write("\n    0x56730bcc: \"core.Null\",")
-            f.write("\n    0x1cb5c415: \"core.Vector\",")
-            f.write("\n    0x73f1f8dc: \"core.MsgContainer\",")
-            f.write("\n    0xae500895: \"core.FutureSalts\",")
-            f.write("\n    0x0949d9dc: \"core.FutureSalt\",")
-            f.write("\n    0x3072cfa1: \"core.GzipPacked\",")
-            f.write("\n    0x5bb8e511: \"core.Message\"")
-
-            f.write("\n}\n")
-
-        for k, v in self.namespaces.items():
-            with open("{}/{}/__init__.py".format(dest, k), "a") as f:
-                f.write("from . import {}\n".format(", ".join([i for i in v])) if v else "")
-
-    @staticmethod
-    def sort_args(args):
-        """Put flags at the end"""
-        args = args.copy()
-        flags = [i for i in args if re.match(r"flags\.\d+\?", i[1])]
-
-        for i in flags:
-            args.remove(i)
-
-        return args + flags
-
-    def compile(self, namespace, name, id, args, has_flags, type):
-        path = "{}/{}/{}".format(dest, self.section, namespace)
         os.makedirs(path, exist_ok=True)
 
         init = "{}/__init__.py".format(path)
 
         if not os.path.exists(init):
             with open(init, "w") as f:
-                f.write(self.notice + "\n\n")
+                f.write(notice + "\n\n")
 
         with open(init, "a") as f:
-            f.write("from .{} import {}\n".format(self.snek(name), self.caml(name)))
+            f.write("from .{} import {}\n".format(snek(c.name), capit(c.name)))
 
-        sorted_args = self.sort_args(args)
+        sorted_args = sort_args(c.args)
-
-        object_id = "0x{}".format(id)
 
         arguments = ", " + ", ".join(
             ["{}{}".format(
                 i[0],
-                "=None" if re.match(r"flags\.\d+\?", i[1]) else ""
+                "=None" if FLAGS_RE.match(i[1]) else ""
             ) for i in sorted_args]
-        ) if args else ""
+        ) if c.args else ""
 
         fields = "\n        ".join(
-            ["self.{0} = {0}  # {1}".format(i[0], i[1]) for i in args]
+            ["self.{0} = {0}  # {1}".format(i[0], i[1]) for i in c.args]
-        ) if args else "pass"
+        ) if c.args else "pass"
 
         docstring_args = []
 
@@ -199,19 +195,24 @@ class Compiler:
                     sub_type = arg_type.split("<")[1][:-1]
 
                     if sub_type in core_types:
-                        arg_type = "List of :obj:`{}`".format(self.caml(sub_type))
+                        if "int" in sub_type or sub_type == "long":
+                            arg_type = "List of :obj:`int`"
+                        elif sub_type == "double":
+                            arg_type = "List of :obj:`float`"
+                        else:
+                            arg_type = "List of :obj:`{}`".format(sub_type.lower())
                     else:
                         arg_type = "List of :class:`pyrogram.api.types.{}`".format(
                             ".".join(
                                 sub_type.split(".")[:-1]
-                                + [self.caml(sub_type.split(".")[-1])]
+                                + [capit(sub_type.split(".")[-1])]
                             )
                         )
                 else:
                     arg_type = ":class:`pyrogram.api.types.{}`".format(
                         ".".join(
                             arg_type.split(".")[:-1]
-                            + [self.caml(arg_type.split(".")[-1])]
+                            + [capit(arg_type.split(".")[-1])]
                         )
                     )
 
@@ -228,26 +229,51 @@ class Compiler:
         else:
             docstring_args = "No parameters required."
 
-        docstring_args = "Attributes:\n        ID (:obj:`int`): ``{}``\n\n    ".format(object_id) + docstring_args
+        docstring_args = "Attributes:\n        ID (:obj:`int`): ``{}``\n\n    ".format(c.id) + docstring_args
 
         docstring_args += "\n\n    Returns:\n        "
-        if type in core_types:
+        if c.return_type in core_types:
-            docstring_args += ":obj:`{}`".format(type.lower())
+            if "int" in c.return_type or c.return_type == "long":
+                return_type = ":obj:`int`"
+            elif c.return_type == "double":
+                return_type = ":obj:`float`"
             else:
-            docstring_args += ":class:`pyrogram.api.types.{}`".format(
+                return_type = ":obj:`{}`".format(c.return_type.lower())
+        else:
+            if c.return_type.startswith("Vector"):
+                sub_type = c.return_type.split("<")[1][:-1]
+
+                if sub_type in core_types:
+                    if "int" in sub_type or sub_type == "long":
+                        return_type = "List of :obj:`int`"
+                    elif sub_type == "double":
+                        return_type = "List of :obj:`float`"
+                    else:
+                        return_type = "List of :obj:`{}`".format(c.return_type.lower())
+                else:
+                    return_type = "List of :class:`pyrogram.api.types.{}`".format(
                         ".".join(
-                    type.split(".")[:-1]
+                            sub_type.split(".")[:-1]
-                    + [self.caml(type.split(".")[-1])]
+                            + [capit(sub_type.split(".")[-1])]
+                        )
+                    )
+            else:
+                return_type = ":class:`pyrogram.api.types.{}`".format(
+                    ".".join(
+                        c.return_type.split(".")[:-1]
+                        + [capit(c.return_type.split(".")[-1])]
                     )
                 )
 
-        if self.section == "functions":
+        docstring_args += return_type
+
+        if c.section == "functions":
             docstring_args += "\n\n    Raises:\n        :class:`pyrogram.Error`"
 
-        if has_flags:
+        if c.has_flags:
             write_flags = []
-            for i in args:
+            for i in c.args:
-                flag = re.match(r"flags\.(\d+)\?", i[1])
+                flag = FLAGS_RE.match(i[1])
                 if flag:
                     write_flags.append("flags |= (1 << {}) if self.{} is not None else 0".format(flag.group(1), i[0]))
 
@@ -259,12 +285,12 @@ class Compiler:
         else:
             write_flags = "# No flags"
 
-        read_flags = "flags = Int.read(b)" if has_flags else "# No flags"
+        read_flags = "flags = Int.read(b)" if c.has_flags else "# No flags"
 
         write_types = read_types = ""
 
-        for arg_name, arg_type in args:
+        for arg_name, arg_type in c.args:
-            flag = re.findall(r"flags\.(\d+)\?([\w<>.]+)", arg_type)
+            flag = FLAGS_RE_2.findall(arg_type)
 
             if flag:
                 index, flag_type = flag[0]
@@ -329,13 +355,13 @@ class Compiler:
                     read_types += "\n        "
                     read_types += "{} = Object.read(b)\n        ".format(arg_name)
 
-        with open("{}/{}.py".format(path, self.snek(name)), "w") as f:
+        with open("{}/{}.py".format(path, snek(c.name)), "w") as f:
             f.write(
-                self.template.format(
+                template.format(
-                    notice=self.notice,
+                    notice=notice,
-                    class_name=self.caml(name),
+                    class_name=capit(c.name),
                     docstring_args=docstring_args,
-                    object_id=object_id,
+                    object_id=c.id,
                     arguments=arguments,
                     fields=fields,
                     read_flags=read_flags,
@@ -346,36 +372,34 @@ class Compiler:
                 )
             )
 
-    @staticmethod
+    with open("{}/all.py".format(DESTINATION), "w") as f:
-    def snek(s):
+        f.write(notice + "\n\n")
-        # https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case
+        f.write("layer = {}\n\n".format(layer))
-        s = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", s)
+        f.write("objects = {")
-        return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s).lower()
 
-    @staticmethod
+        for c in combinators:
-    def caml(s):
+            path = ".".join(filter(None, [c.section, c.namespace, capit(c.name)]))
-        s = Compiler.snek(s).split("_")
+            f.write("\n    {}: \"{}\",".format(c.id, path))
-        return "".join([str(i.title()) for i in s])
 
-    def start(self):
+        f.write("\n    0xbc799737: \"core.BoolFalse\",")
-        shutil.rmtree("{}/types".format(dest), ignore_errors=True)
+        f.write("\n    0x997275b5: \"core.BoolTrue\",")
-        shutil.rmtree("{}/functions".format(dest), ignore_errors=True)
+        f.write("\n    0x56730bcc: \"core.Null\",")
+        f.write("\n    0x1cb5c415: \"core.Vector\",")
+        f.write("\n    0x73f1f8dc: \"core.MsgContainer\",")
+        f.write("\n    0xae500895: \"core.FutureSalts\",")
+        f.write("\n    0x0949d9dc: \"core.FutureSalt\",")
+        f.write("\n    0x3072cfa1: \"core.GzipPacked\",")
+        f.write("\n    0x5bb8e511: \"core.Message\"")
 
-        self.read_schema()
+        f.write("\n}\n")
-        self.parse_schema()
-        self.finish()
 
-        print()
+    for k, v in namespaces.items():
-
+        with open("{}/{}/__init__.py".format(DESTINATION, k), "a") as f:
-
+            f.write("from . import {}\n".format(", ".join([i for i in v])) if v else "")
-def start():
-    c = Compiler()
-    c.start()
 
 
 if "__main__" == __name__:
-    home = "."
+    HOME = "."
-    dest = "../../pyrogram/api"
+    DESTINATION = "../../pyrogram/api"
     notice_path = "../../NOTICE"
-
     start()