Xtao-Labs/MTPyroger
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add API compiler

Browse Source
This commit is contained in:
Dan 2017-12-05 12:16:39 +01:00
parent 45ee7aa8b1
commit 28704b9326
7 changed files with 1543 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
compiler/__init__.py Normal file
View File

@ -0,0 +1,17 @@
# Pyrogram - Telegram MTProto API Client Library for Python
# Copyright (C) 2017 Dan Tès <https://github.com/delivrance>
#
# This file is part of Pyrogram.
#
# Pyrogram is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pyrogram is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Pyrogram. If not, see <http://www.gnu.org/licenses/>.

17
compiler/api/__init__.py Normal file
View File

@ -0,0 +1,17 @@
# Pyrogram - Telegram MTProto API Client Library for Python
# Copyright (C) 2017 Dan Tès <https://github.com/delivrance>
#
# This file is part of Pyrogram.
#
# Pyrogram is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pyrogram is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Pyrogram. If not, see <http://www.gnu.org/licenses/>.

313
compiler/api/compiler.py Normal file
View File

@ -0,0 +1,313 @@
# Pyrogram - Telegram MTProto API Client Library for Python
# Copyright (C) 2017 Dan Tès <https://github.com/delivrance>
#
# This file is part of Pyrogram.
#
# Pyrogram is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pyrogram is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Pyrogram. If not, see <http://www.gnu.org/licenses/>.
import os
import re
import shutil
home = "compiler/api"
dest = "pyrogram/api"
notice_path = "NOTICE"
core_types = ["int", "long", "int128", "int256", "double", "bytes", "string", "Bool"]
# TODO: Compiler was written in a rush. variables/methods name and pretty much all the code is fuzzy, but it works
# TODO: Some constructors have flags:# but not flags.\d+\?
class Compiler:
def __init__(self):
self.section = "types" # TL Schema starts with types
self.namespaces = {"types": set(), "functions": set()}
self.objects = {}
self.layer = None
self.schema = None
with open("{}/template/class.txt".format(home)) as f:
self.template = f.read()
with open(notice_path) as f:
notice = []
for line in f.readlines():
notice.append("# {}".format(line).strip())
self.notice = "\n".join(notice)
def read_schema(self):
"""Read schema files"""
with open("{}/source/auth_key.tl".format(home)) as auth, \
open("{}/source/sys_msgs.tl".format(home)) as system, \
open("{}/source/main_api.tl".format(home)) as api:
self.schema = auth.read() + system.read() + api.read()
def parse_schema(self):
"""Parse schema line by line"""
total = len(self.schema.splitlines())
for i, line in enumerate(self.schema.splitlines()):
# Check for section changer lines
section = re.match(r"---(\w+)---", line)
if section:
self.section = section.group(1)
continue
# Save the layer version
layer = re.match(r"//\sLAYER\s(\d+)", line)
if layer:
self.layer = layer.group(1)
continue
combinator = re.match(r"^([\w.]+)#([0-9a-f]+)\s(.*)=\s([\w<>.]+);$", line, re.MULTILINE)
if combinator:
name, id, args, type = combinator.groups()
namespace, name = name.split(".") if "." in name else ("", name)
args = re.findall(r"[^{](\w+):([\w?!.<>]+)", line)
print("Compiling APIs: [{}%]".format(round(i * 100 / total)), end="\r")
for i in args:
if re.match(r"flags\.\d+\?", i[1]):
has_flags = True
break
else:
has_flags = False
if name == "updates":
name = "update"
for i, item in enumerate(args):
# if item[0] in keyword.kwlist + dir(builtins) + ["self"]:
if item[0] == "self":
args[i] = ("is_{}".format(item[0]), item[1])
if namespace:
self.namespaces[self.section].add(namespace)
self.compile(namespace, name, id, args, has_flags)
self.objects[id] = "{}.{}{}.{}".format(
self.section,
"{}.".format(namespace) if namespace else "",
self.snek(name),
self.caml(name)
)
def finish(self):
with open("{}/all.py".format(dest), "w") as f:
f.write(self.notice + "\n\n")
f.write("layer = {}\n\n".format(self.layer))
f.write("objects = {")
for k, v in self.objects.items():
v = v.split(".")
del v[-2]
v = ".".join(v)
f.write("\n 0x{}: \"{}\",".format(k.zfill(8), v))
f.write("\n 0xbc799737: \"core.BoolFalse\",")
f.write("\n 0x997275b5: \"core.BoolTrue\",")
f.write("\n 0x1cb5c415: \"core.Vector\",")
f.write("\n 0x73f1f8dc: \"core.MsgContainer\",")
f.write("\n 0xae500895: \"core.FutureSalts\",")
f.write("\n 0x0949d9dc: \"core.FutureSalt\",")
f.write("\n 0x3072cfa1: \"core.GzipPacked\",")
f.write("\n 0x5bb8e511: \"core.Message\"")
f.write("\n}\n")
for k, v in self.namespaces.items():
with open("{}/{}/__init__.py".format(dest, k), "a") as f:
f.write("from . import {}\n".format(", ".join([i for i in v])) if v else "")
@staticmethod
def sort_args(args):
"""Put flags at the end"""
args = args.copy()
flags = [i for i in args if re.match(r"flags\.\d+\?", i[1])]
for i in flags:
args.remove(i)
return args + flags
def compile(self, namespace, name, id, args, has_flags):
path = "{}/{}/{}".format(dest, self.section, namespace)
os.makedirs(path, exist_ok=True)
init = "{}/__init__.py".format(path)
if not os.path.exists(init):
with open(init, "w") as f:
f.write(self.notice + "\n\n")
with open(init, "a") as f:
f.write("from .{} import {}\n".format(self.snek(name), self.caml(name)))
sorted_args = self.sort_args(args)
object_id = "0x{}".format(id)
arguments = ", " + ", ".join(
["{}{}".format(
i[0],
"=None" if re.match(r"flags\.\d+\?", i[1]) else ""
) for i in sorted_args]
) if args else ""
fields = "\n ".join(
["self.{0} = {0} # {1}".format(i[0], i[1]) for i in args]
) if args else "pass"
if has_flags:
write_flags = []
for i in args:
flag = re.match(r"flags\.(\d+)\?", i[1])
if flag:
write_flags.append("flags |= (1 << {}) if self.{} is not None else 0".format(flag.group(1), i[0]))
write_flags = "\n ".join([
"flags = 0",
"\n ".join(write_flags),
"b.write(Int(flags))"
])
else:
write_flags = "# No flags"
read_flags = "flags = Int.read(b)" if has_flags else "# No flags"
write_types = read_types = ""
for arg_name, arg_type in args:
flag = re.findall(r"flags\.(\d+)\?([\w<>.]+)", arg_type)
if flag:
index, flag_type = flag[0]
if flag_type == "true":
read_types += "\n "
read_types += "{} = True if flags & (1 << {}) else False".format(arg_name, index)
elif flag_type in core_types:
write_types += "\n "
write_types += "if self.{} is not None:\n ".format(arg_name)
write_types += "b.write({}(self.{}))\n ".format(flag_type.title(), arg_name)
read_types += "\n "
read_types += "{} = {}.read(b) if flags & (1 << {}) else None".format(
arg_name, flag_type.title(), index
)
elif "vector" in flag_type.lower():
sub_type = arg_type.split("<")[1][:-1]
write_types += "\n "
write_types += "if self.{} is not None:\n ".format(arg_name)
write_types += "b.write(Vector(self.{}{}))\n ".format(
arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else ""
)
read_types += "\n "
read_types += "{} = Object.read(b{}) if flags & (1 << {}) else []\n ".format(
arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else "", index
)
else:
write_types += "\n "
write_types += "if self.{} is not None:\n ".format(arg_name)
write_types += "b.write(self.{}.write())\n ".format(arg_name)
read_types += "\n "
read_types += "{} = Object.read(b) if flags & (1 << {}) else None\n ".format(
arg_name, index
)
else:
if arg_type in core_types:
write_types += "\n "
write_types += "b.write({}(self.{}))\n ".format(arg_type.title(), arg_name)
read_types += "\n "
read_types += "{} = {}.read(b)\n ".format(arg_name, arg_type.title())
elif "vector" in arg_type.lower():
sub_type = arg_type.split("<")[1][:-1]
write_types += "\n "
write_types += "b.write(Vector(self.{}{}))\n ".format(
arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else ""
)
read_types += "\n "
read_types += "{} = Object.read(b{})\n ".format(
arg_name, ", {}".format(sub_type.title()) if sub_type in core_types else ""
)
else:
write_types += "\n "
write_types += "b.write(self.{}.write())\n ".format(arg_name)
read_types += "\n "
read_types += "{} = Object.read(b)\n ".format(arg_name)
with open("{}/{}.py".format(path, self.snek(name)), "w") as f:
f.write(
self.template.format(
notice=self.notice,
class_name=self.caml(name),
object_id=object_id,
arguments=arguments,
fields=fields,
read_flags=read_flags,
read_types=read_types,
write_flags=write_flags,
write_types=write_types,
return_arguments=", ".join([i[0] for i in sorted_args])
)
)
@staticmethod
def snek(s):
# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case
s = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", s)
return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s).lower()
@staticmethod
def caml(s):
s = Compiler.snek(s).split("_")
return "".join([str(i.title()) for i in s])
def start(self):
shutil.rmtree("{}/types".format(dest), ignore_errors=True)
shutil.rmtree("{}/functions".format(dest), ignore_errors=True)
self.read_schema()
self.parse_schema()
self.finish()
print()
def start():
c = Compiler()
c.start()
if "__main__" == __name__:
home = "."
dest = "../../pyrogram/api"
notice_path = "../../NOTICE"
start()

37
compiler/api/source/auth_key.tl Normal file
View File

@ -0,0 +1,37 @@
///////////////////////////////
/// Authorization key creation
///////////////////////////////
---types---
resPQ#05162463 nonce:int128 server_nonce:int128 pq:bytes server_public_key_fingerprints:Vector<long> = ResPQ;
p_q_inner_data#83c95aec pq:bytes p:bytes q:bytes nonce:int128 server_nonce:int128 new_nonce:int256 = P_Q_inner_data;
p_q_inner_data_temp#3c6a84d4 pq:bytes p:bytes q:bytes nonce:int128 server_nonce:int128 new_nonce:int256 expires_in:int = P_Q_inner_data;
bind_auth_key_inner#75a3f765 nonce:long temp_auth_key_id:long perm_auth_key_id:long temp_session_id:long expires_at:int = BindAuthKeyInner;
server_DH_params_fail#79cb045d nonce:int128 server_nonce:int128 new_nonce_hash:int128 = Server_DH_Params;
server_DH_params_ok#d0e8075c nonce:int128 server_nonce:int128 encrypted_answer:bytes = Server_DH_Params;
server_DH_inner_data#b5890dba nonce:int128 server_nonce:int128 g:int dh_prime:bytes g_a:bytes server_time:int = Server_DH_inner_data;
client_DH_inner_data#6643b654 nonce:int128 server_nonce:int128 retry_id:long g_b:bytes = Client_DH_Inner_Data;
dh_gen_ok#3bcbf734 nonce:int128 server_nonce:int128 new_nonce_hash1:int128 = Set_client_DH_params_answer;
dh_gen_retry#46dc1fb9 nonce:int128 server_nonce:int128 new_nonce_hash2:int128 = Set_client_DH_params_answer;
dh_gen_fail#a69dae02 nonce:int128 server_nonce:int128 new_nonce_hash3:int128 = Set_client_DH_params_answer;
destroy_auth_key_ok#f660e1d4 = DestroyAuthKeyRes;
destroy_auth_key_none#0a9f2259 = DestroyAuthKeyRes;
destroy_auth_key_fail#ea109b13 = DestroyAuthKeyRes;
---functions---
req_pq#60469778 nonce:int128 = ResPQ;
req_DH_params#d712e4be nonce:int128 server_nonce:int128 p:bytes q:bytes public_key_fingerprint:long encrypted_data:bytes = Server_DH_Params;
set_client_DH_params#f5045f1f nonce:int128 server_nonce:int128 encrypted_data:bytes = Set_client_DH_params_answer;
destroy_auth_key#d1435160 = DestroyAuthKeyRes;

1071
compiler/api/source/main_api.tl Normal file
View File

File diff suppressed because it is too large Load Diff

62
compiler/api/source/sys_msgs.tl Normal file
View File

@ -0,0 +1,62 @@
///////////////////////////////
////////////// System messages
///////////////////////////////
---types---
msgs_ack#62d6b459 msg_ids:Vector<long> = MsgsAck;
bad_msg_notification#a7eff811 bad_msg_id:long bad_msg_seqno:int error_code:int = BadMsgNotification;
bad_server_salt#edab447b bad_msg_id:long bad_msg_seqno:int error_code:int new_server_salt:long = BadMsgNotification;
msgs_state_req#da69fb52 msg_ids:Vector<long> = MsgsStateReq;
msgs_state_info#04deb57d req_msg_id:long info:string = MsgsStateInfo;
msgs_all_info#8cc0d131 msg_ids:Vector<long> info:string = MsgsAllInfo;
msg_detailed_info#276d3ec6 msg_id:long answer_msg_id:long bytes:int status:int = MsgDetailedInfo;
msg_new_detailed_info#809db6df answer_msg_id:long bytes:int status:int = MsgDetailedInfo;
msg_resend_req#7d861a08 msg_ids:Vector<long> = MsgResendReq;
msg_resend_ans_req#8610baeb msg_ids:Vector<long> = MsgResendReq;
rpc_result#f35c6d01 req_msg_id:long result:Object = RpcResult;
rpc_error#2144ca19 error_code:int error_message:string = RpcError;
rpc_answer_unknown#5e2ad36e = RpcDropAnswer;
rpc_answer_dropped_running#cd78e586 = RpcDropAnswer;
rpc_answer_dropped#a43ad8b7 msg_id:long seq_no:int bytes:int = RpcDropAnswer;
// future_salt#0949d9dc valid_since:int valid_until:int salt:long = FutureSalt; // Parsed manually
// future_salts#ae500895 req_msg_id:long now:int salts:vector<future_salt> = FutureSalts; // Parsed manually
pong#347773c5 msg_id:long ping_id:long = Pong;
destroy_session_ok#e22045fc session_id:long = DestroySessionRes;
destroy_session_none#62d350c9 session_id:long = DestroySessionRes;
new_session_created#9ec20908 first_msg_id:long unique_id:long server_salt:long = NewSession;
//// 0x5bb8e511 = hex(crc32(b"message msg_id:long seqno:int bytes:int body:Object = Message"))
// message#5bb8e511 msg_id:long seqno:int bytes:int body:Object = Message; // Parsed manually
// msg_container#73f1f8dc messages:vector<message> = MessageContainer; // Parsed manually
// msg_copy#e06046b2 orig_message:Message = MessageCopy; // Not used
// gzip_packed#3072cfa1 packed_data:string = Object; // Parsed manually
//// 0xd433ad73 = hex(crc32(b"ipPort ipv4:int port:int = IpPort"))
ipPort#d433ad73 ipv4:int port:int = IpPort;
help.configSimple#d997c3c5 date:int expires:int dc_id:int ip_port_list:Vector<ipPort> = help.ConfigSimple;
---functions---
rpc_drop_answer#58e4a740 req_msg_id:long = RpcDropAnswer;
get_future_salts#b921bd04 num:int = FutureSalts;
ping#7abe77ec ping_id:long = Pong;
ping_delay_disconnect#f3427b8c ping_id:long disconnect_delay:int = Pong;
destroy_session#e7512126 session_id:long = DestroySessionRes;
contest.saveDeveloperInfo#9a5f6e95 vk_id:int name:string phone_number:string age:int city:string = Bool;
http_wait#9299359f max_delay:int wait_after:int max_wait:int = HttpWait;

26
compiler/api/template/class.txt Normal file
View File

@ -0,0 +1,26 @@
{notice}
from io import BytesIO
from pyrogram.api.core import *
class {class_name}(Object):
ID = {object_id}
def __init__(self{arguments}):
{fields}
@staticmethod
def read(b: BytesIO) -> "{class_name}":
{read_flags}
{read_types}
return {class_name}({return_arguments})
def write(self) -> bytes:
b = BytesIO()
b.write(Int(self.ID, False))
{write_flags}
{write_types}
return b.getvalue()