Port tl-gen from grammers

generator/tests/definition_test.py

@@ -0,0 +1,200 @@
+from pytest import mark, raises
+from telethon_generator.tl_parser import (
+    Definition,
+    Flag,
+    FlagsParameter,
+    NormalParameter,
+    Parameter,
+    Type,
+)
+
+
+def test_parse_empty_def() -> None:
+    with raises(ValueError) as e:
+        Definition.from_str("")
+    e.match("empty")
+
+
+@mark.parametrize("defn", ["foo#bar = baz", "foo#? = baz", "foo# = baz"])
+def test_parse_bad_id(defn: str) -> None:
+    with raises(ValueError) as e:
+        Definition.from_str(defn)
+    e.match("invalid id")
+
+
+def test_parse_no_name() -> None:
+    with raises(ValueError) as e:
+        Definition.from_str(" = foo")
+    e.match("missing name")
+
+
+@mark.parametrize("defn", ["foo", "foo ="])
+def test_parse_no_type(defn: str) -> None:
+    with raises(ValueError) as e:
+        Definition.from_str(defn)
+    e.match("missing type")
+
+
+def test_parse_unimplemented() -> None:
+    with raises(ValueError) as e:
+        Definition.from_str("int ? = Int")
+    e.match("not implemented")
+
+
+@mark.parametrize(
+    ("defn", "id"),
+    [
+        (
+            "rpc_answer_dropped msg_id:long seq_no:int bytes:int = RpcDropAnswer",
+            0xA43AD8B7,
+        ),
+        (
+            "rpc_answer_dropped#123456 msg_id:long seq_no:int bytes:int = RpcDropAnswer",
+            0x123456,
+        ),
+    ],
+)
+def test_parse_override_id(defn: str, id: int) -> None:
+    assert Definition.from_str(defn).id == id
+
+
+def test_parse_valid_definition() -> None:
+    defn = Definition.from_str("a#1=d")
+    assert defn.name == "a"
+    assert defn.id == 1
+    assert len(defn.params) == 0
+    assert defn.ty == Type(
+        namespace=[],
+        name="d",
+        bare=True,
+        generic_ref=False,
+        generic_arg=None,
+    )
+
+    defn = Definition.from_str("a=d<e>")
+    assert defn.name == "a"
+    assert defn.id != 0
+    assert len(defn.params) == 0
+    assert defn.ty == Type(
+        namespace=[],
+        name="d",
+        bare=True,
+        generic_ref=False,
+        generic_arg=Type.from_str("e"),
+    )
+
+    defn = Definition.from_str("a b:c = d")
+    assert defn.name == "a"
+    assert defn.id != 0
+    assert len(defn.params) == 1
+    assert defn.ty == Type(
+        namespace=[],
+        name="d",
+        bare=True,
+        generic_ref=False,
+        generic_arg=None,
+    )
+
+    defn = Definition.from_str("a#1 {b:Type} c:!b = d")
+    assert defn.name, "a"
+    assert defn.id, 1
+    assert len(defn.params), 1
+    assert isinstance(defn.params[0].ty, NormalParameter)
+    assert defn.params[0].ty.ty.generic_ref
+    assert defn.ty == Type(
+        namespace=[],
+        name="d",
+        bare=True,
+        generic_ref=False,
+        generic_arg=None,
+    )
+
+
+def test_parse_multiline_definition() -> None:
+    defn = """
+        first#1 lol:param
+            = t;
+        """
+
+    assert Definition.from_str(defn).id, 1
+
+    defn = """
+        second#2
+            lol:String
+        = t;
+        """
+
+    assert Definition.from_str(defn).id, 2
+
+    defn = """
+        third#3
+
+            lol:String
+
+        =
+                    t;
+        """
+
+    assert Definition.from_str(defn).id, 3
+
+
+def test_parse_complete() -> None:
+    defn = "ns1.name#123 {X:Type} flags:# pname:flags.10?ns2.Vector<!X> = ns3.Type"
+    assert Definition.from_str(defn) == Definition(
+        namespace=["ns1"],
+        name="name",
+        id=0x123,
+        params=[
+            Parameter(
+                name="flags",
+                ty=FlagsParameter(),
+            ),
+            Parameter(
+                name="pname",
+                ty=NormalParameter(
+                    ty=Type(
+                        namespace=["ns2"],
+                        name="Vector",
+                        bare=False,
+                        generic_ref=False,
+                        generic_arg=Type(
+                            namespace=[],
+                            name="X",
+                            bare=False,
+                            generic_ref=True,
+                            generic_arg=None,
+                        ),
+                    ),
+                    flag=Flag(name="flags", index=10),
+                ),
+            ),
+        ],
+        ty=Type(
+            namespace=["ns3"],
+            name="Type",
+            bare=False,
+            generic_ref=False,
+            generic_arg=None,
+        ),
+    )
+
+
+@mark.parametrize(
+    "defn",
+    [
+        "name param:!X = Type",
+        "name {X:Type} param:!Y = Type",
+        "name param:flags.0?true = Type",
+        "name foo:# param:flags.0?true = Type",
+    ],
+)
+def test_parse_missing_def(defn: str) -> None:
+    with raises(ValueError) as e:
+        Definition.from_str(defn)
+
+    e.match("missing def")
+
+
+def test_test_to_string() -> None:
+    defn = "ns1.name#123 {X:Type} flags:# pname:flags.10?ns2.Vector<!X> = ns3.Type"
+    assert str(Definition.from_str(defn)), defn

generator/tests/generator_test.py

@@ -0,0 +1,82 @@
+from typing import List, Optional
+
+from telethon_generator.codegen import FakeFs, ParsedTl, generate
+from telethon_generator.tl_parser import Definition, parse_tl_file
+
+
+def get_definitions(contents: str) -> List[Definition]:
+    return [defn for defn in parse_tl_file(contents) if not isinstance(defn, Exception)]
+
+
+def gen_py_code(
+    *,
+    typedefs: Optional[List[Definition]] = None,
+    functiondefs: Optional[List[Definition]] = None,
+) -> str:
+    fs = FakeFs()
+    generate(
+        fs, ParsedTl(layer=0, typedefs=typedefs or [], functiondefs=functiondefs or [])
+    )
+    generated = bytearray()
+    for path, data in fs._files.items():
+        if path.stem not in ("__init__", "layer"):
+            generated += f"# {path}\n".encode("utf-8")
+            generated += data
+            data += b"\n"
+    return str(generated, "utf-8")
+
+
+def test_generic_functions_use_bytes_parameters() -> None:
+    definitions = get_definitions(
+        "invokeWithLayer#da9b0d0d {X:Type} layer:int query:!X = X;"
+    )
+    result = gen_py_code(functiondefs=definitions)
+    assert "invoke_with_layer" in result
+    assert "query: bytes" in result
+    assert "buffer += query" in result
+
+
+def test_recursive_direct() -> None:
+    definitions = get_definitions("textBold#6724abc4 text:RichText = RichText;")
+    result = gen_py_code(typedefs=definitions)
+    assert "text: abcs.RichText" in result
+    assert "read_serializable" in result
+    assert "write_boxed_to" in result
+
+
+def test_recursive_indirect() -> None:
+    definitions = get_definitions(
+        """
+        messageExtendedMedia#ee479c64 media:MessageMedia = MessageExtendedMedia;
+        messageMediaInvoice#f6a548d3 flags:# extended_media:flags.4?MessageExtendedMedia = MessageMedia;
+        """
+    )
+    result = gen_py_code(typedefs=definitions)
+    assert "media: abcs.MessageMedia" in result
+    assert "extended_media: Optional[abcs.MessageExtendedMedia])" in result
+    assert "write_boxed_to" in result
+    assert "._write_to" not in result
+    assert "read_serializable" in result
+
+
+def test_recursive_no_hang() -> None:
+    definitions = get_definitions(
+        """
+        inputUserFromMessage#1da448e2 peer:InputPeer msg_id:int user_id:long = InputUser;
+        inputPeerUserFromMessage#a87b0a1c peer:InputPeer msg_id:int user_id:long = InputPeer;
+        """
+    )
+    gen_py_code(typedefs=definitions)
+
+
+def test_recursive_vec() -> None:
+    definitions = get_definitions(
+        """
+        jsonObjectValue#c0de1bd9 key:string value:JSONValue = JSONObjectValue;
+
+        jsonArray#f7444763 value:Vector<JSONValue> = JSONValue;
+        jsonObject#99c1d49d value:Vector<JSONObjectValue> = JSONValue;
+        """
+    )
+    result = gen_py_code(typedefs=definitions)
+    assert "value: List[abcs.JSONObjectValue]" in result

generator/tests/parameter_test.py

@@ -0,0 +1,100 @@
+from pytest import mark, raises
+from telethon_generator.tl_parser import (
+    Flag,
+    FlagsParameter,
+    NormalParameter,
+    Parameter,
+    Type,
+    TypeDefNotImplemented,
+)
+
+
+@mark.parametrize("param", [":noname", "notype:", ":"])
+def test_empty_param(param: str) -> None:
+    with raises(ValueError) as e:
+        Parameter.from_str(param)
+    e.match("empty")
+
+
+@mark.parametrize("param", ["", "no colon", "colonless"])
+def test_unknown_param(param: str) -> None:
+    with raises(ValueError) as e:
+        Parameter.from_str(param)
+    e.match("not implemented")
+
+
+@mark.parametrize("param", ["foo:bar?", "foo:?bar", "foo:bar?baz", "foo:bar.baz?qux"])
+def test_bad_flags(param: str) -> None:
+    with raises(ValueError) as e:
+        Parameter.from_str(param)
+    e.match("invalid flag")
+
+
+@mark.parametrize("param", ["foo:<bar", "foo:bar<"])
+def test_bad_generics(param: str) -> None:
+    with raises(ValueError) as e:
+        Parameter.from_str(param)
+    e.match("invalid generic")
+
+
+def test_type_def_param() -> None:
+    with raises(TypeDefNotImplemented) as e:
+        Parameter.from_str("{a:Type}")
+    e.match("typedef not implemented: a")
+
+
+def test_unknown_def_param() -> None:
+    with raises(ValueError) as e:
+        Parameter.from_str("{a:foo}")
+    e.match("missing def")
+
+
+def test_valid_param() -> None:
+    assert Parameter.from_str("foo:#") == Parameter(name="foo", ty=FlagsParameter())
+    assert Parameter.from_str("foo:!bar") == Parameter(
+        name="foo",
+        ty=NormalParameter(
+            ty=Type(
+                namespace=[], name="bar", bare=True, generic_ref=True, generic_arg=None
+            ),
+            flag=None,
+        ),
+    )
+    assert Parameter.from_str("foo:bar.1?baz") == Parameter(
+        name="foo",
+        ty=NormalParameter(
+            ty=Type(
+                namespace=[], name="baz", bare=True, generic_ref=False, generic_arg=None
+            ),
+            flag=Flag(
+                name="bar",
+                index=1,
+            ),
+        ),
+    )
+    assert Parameter.from_str("foo:bar<baz>") == Parameter(
+        name="foo",
+        ty=NormalParameter(
+            ty=Type(
+                namespace=[],
+                name="bar",
+                bare=True,
+                generic_ref=False,
+                generic_arg=Type.from_str("baz"),
+            ),
+            flag=None,
+        ),
+    )
+    assert Parameter.from_str("foo:bar.1?baz<qux>") == Parameter(
+        name="foo",
+        ty=NormalParameter(
+            ty=Type(
+                namespace=[],
+                name="baz",
+                bare=True,
+                generic_ref=False,
+                generic_arg=Type.from_str("qux"),
+            ),
+            flag=Flag(name="bar", index=1),
+        ),
+    )

generator/tests/tl_iterator_test.py

@@ -0,0 +1,28 @@
+from pytest import raises
+from telethon_generator.tl_parser import FunctionDef, TypeDef, parse_tl_file
+
+
+def test_parse_bad_separator() -> None:
+    with raises(ValueError) as e:
+        for _ in parse_tl_file("---foo---"):
+            pass
+    e.match("bad separator")
+
+
+def test_parse_file() -> None:
+    items = list(
+        parse_tl_file(
+            """
+            // leading; comment
+            first#1 = t; // inline comment
+            ---functions---
+            second and bad;
+            third#3 = t;
+            // trailing comment
+            """
+        )
+    )
+    assert len(items) == 3
+    assert isinstance(items[0], TypeDef) and items[0].id == 1
+    assert isinstance(items[1], ValueError)
+    assert isinstance(items[2], FunctionDef) and items[2].id == 3

generator/tests/ty_test.py

@@ -0,0 +1,90 @@
+from typing import Optional
+
+from pytest import mark, raises
+from telethon_generator.tl_parser import Type
+
+
+def test_empty_simple() -> None:
+    with raises(ValueError) as e:
+        Type.from_str("")
+    e.match("empty")
+
+
+def test_simple() -> None:
+    assert Type.from_str("foo") == Type(
+        namespace=[], name="foo", bare=True, generic_ref=False, generic_arg=None
+    )
+
+
+@mark.parametrize("ty", [".", "..", ".foo", "foo.", "foo..foo", ".foo."])
+def test_check_empty_namespaced(ty: str) -> None:
+    with raises(ValueError) as e:
+        Type.from_str(ty)
+        e.match("empty")
+
+
+def test_check_namespaced() -> None:
+    assert Type.from_str("foo.bar.baz") == Type(
+        namespace=["foo", "bar"],
+        name="baz",
+        bare=True,
+        generic_ref=False,
+        generic_arg=None,
+    )
+
+
+@mark.parametrize(
+    "ty",
+    [
+        "foo",
+        "Foo.bar",
+        "!bar",
+    ],
+)
+def test_bare(ty: str) -> None:
+    assert Type.from_str(ty).bare
+
+
+@mark.parametrize(
+    "ty",
+    [
+        "Foo",
+        "Foo.Bar",
+        "!foo.Bar",
+    ],
+)
+def test_bare_not(ty: str) -> None:
+    assert not Type.from_str(ty).bare
+
+
+@mark.parametrize(
+    "ty",
+    [
+        "!f",
+        "!Foo",
+        "!X",
+    ],
+)
+def test_generic_ref(ty: str) -> None:
+    assert Type.from_str(ty).generic_ref
+
+
+def test_generic_ref_not() -> None:
+    assert not Type.from_str("f").generic_ref
+
+
+@mark.parametrize(
+    ("ty", "generic"),
+    [
+        ("foo.bar", None),
+        ("foo<bar>", "bar"),
+        ("foo<bar.Baz>", "bar.Baz"),
+        ("foo<!bar.Baz>", "!bar.Baz"),
+        ("foo<bar<baz>>", "bar<baz>"),
+    ],
+)
+def test_generic_arg(ty: str, generic: Optional[str]) -> None:
+    if generic is None:
+        assert Type.from_str(ty).generic_arg is None
+    else:
+        assert Type.from_str(ty).generic_arg == Type.from_str(generic)

generator/tests/utils_test.py

@@ -0,0 +1,41 @@
+from telethon_generator._impl.tl_parser.utils import infer_id, remove_tl_comments
+
+
+def test_remove_comments_noop() -> None:
+    data = "hello\nworld"
+    assert remove_tl_comments(data) == data
+
+    data = " \nhello\nworld\n "
+    assert remove_tl_comments(data) == data
+
+
+def test_remove_comments_leading() -> None:
+    input = " // hello\n world "
+    expected = " \n world "
+    assert remove_tl_comments(input) == expected
+
+
+def test_remove_comments_trailing() -> None:
+    input = " \nhello \n // world \n \n "
+    expected = " \nhello \n \n \n "
+    assert remove_tl_comments(input) == expected
+
+
+def test_remove_comments_many() -> None:
+    input = "no\n//yes\nno\n//yes\nno\n"
+    expected = "no\n\nno\n\nno\n"
+    assert remove_tl_comments(input) == expected
+
+
+def test_check_infer_id() -> None:
+    defn = "rpc_answer_dropped msg_id:long seq_no:int bytes:int = RpcDropAnswer"
+    assert infer_id(defn) == 0xA43AD8B7
+
+    defn = "msgs_ack msg_ids:Vector<long> = MsgsAck"
+    assert infer_id(defn) == 0x62D6B459
+
+    defn = "invokeAfterMsg {X:Type} msg_id:long query:!X = X"
+    assert infer_id(defn) == 0xCB9F372D
+
+    defn = "inputMessagesFilterPhoneCalls flags:# missed:flags.0?true = MessagesFilter"
+    assert infer_id(defn) == 0x80C99768