From 3ab03b9c3bcd363957409801316c90a2d27d6502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Fri, 3 Mar 2023 20:36:44 -0800 Subject: [PATCH] Python compat fixes --- codon/parser/cache.cpp | 33 ++++--- codon/parser/visitors/typecheck/call.cpp | 11 ++- codon/parser/visitors/typecheck/loops.cpp | 4 +- stdlib/internal/python.codon | 101 ++++++++++++++-------- 4 files changed, 95 insertions(+), 54 deletions(-) diff --git a/codon/parser/cache.cpp b/codon/parser/cache.cpp index a89da2cc..3da5e68d 100644 --- a/codon/parser/cache.cpp +++ b/codon/parser/cache.cpp @@ -268,8 +268,12 @@ void Cache::populatePythonModule() { }; const std::string pyWrap = "std.internal.python._PyWrap"; - for (const auto &[cn, c] : classes) - if (c.module.empty() && startswith(cn, "Pyx")) { + for (const auto &[cn, c] : classes) { + if (c.module.empty()) { + if (!in(c.methods, "__to_py__") || !in(c.methods, "__from_py__")) + continue; + + LOG("[py] Cythonizing {}", cn); ir::PyType py{rev(cn), c.ast->getDocstr()}; auto tc = typeCtx->forceFind(cn)->type; @@ -284,8 +288,6 @@ void Cache::populatePythonModule() { auto &fna = functions[fnn].ast; fna->getFunction()->suite = N(N( N(pyWrap + ".wrap_to_py:0"), N(fna->args[0].name))); - } else { - compilationError(fmt::format("class '{}' has no __to_py__"), rev(cn)); } if (auto ofnn = in(c.methods, "__from_py__")) { auto fnn = overloads[*ofnn].begin()->name; // default first overload! @@ -293,8 +295,6 @@ void Cache::populatePythonModule() { fna->getFunction()->suite = N(N(N(pyWrap + ".wrap_from_py:0"), N(fna->args[0].name), N(cn))); - } else { - compilationError(fmt::format("class '{}' has no __from_py__"), rev(cn)); } for (auto &n : std::vector{"__from_py__", "__to_py__"}) { auto fnn = overloads[*in(c.methods, n)].begin()->name; @@ -327,12 +327,11 @@ void Cache::populatePythonModule() { if (overloads[ofnn].size() == 1 && functions[canonicalName].ast->hasAttr("autogenerated")) continue; - auto fna = functions[canonicalName].ast; bool isMethod = fna->hasAttr(Attr::Method); - std::string call = pyWrap + ".wrap_single"; - if (fna->args.size() - isMethod > 1) - call = pyWrap + ".wrap_multiple"; + std::string call = pyWrap + ".wrap_multiple"; + if (isMethod) + call += "_method"; bool isMagic = false; if (startswith(n, "__") && endswith(n, "__")) { if (auto i = in(classes[pyWrap].methods, @@ -347,12 +346,12 @@ void Cache::populatePythonModule() { auto generics = std::vector{tc}; if (!isMagic) { generics.push_back(std::make_shared(this, n)); - generics.push_back(std::make_shared(this, isMethod)); } auto f = realizeIR(functions[fnName].type, generics); if (!f) continue; + LOG("[py] {} -> {}", n, call); if (n == "__repr__") { py.repr = f; } else if (n == "__add__") { @@ -450,7 +449,9 @@ void Cache::populatePythonModule() { ir::PyFunction{n, fna->getDocstr(), f, fna->hasAttr(Attr::Method) ? ir::PyFunction::Type::METHOD : ir::PyFunction::Type::CLASS, - int(fna->args.size()) - fna->hasAttr(Attr::Method)}); + // always use FASTCALL for now; works even for 0- or 1- arg methods + 2 + }); } } @@ -480,9 +481,10 @@ void Cache::populatePythonModule() { } pyModule->types.push_back(py); } + } // Handle __iternext__ wrappers - auto cin = "std.internal.python._PyWrap.IterWrap"; + auto cin = "_PyWrap.IterWrap"; for (auto &[cn, cr] : classes[cin].realizations) { LOG("[py] iterfn: {}", cn); ir::PyType py{cn, ""}; @@ -519,7 +521,6 @@ void Cache::populatePythonModule() { call = pyWrap + ".wrap_multiple"; auto fnName = call + ":0"; seqassertn(in(functions, fnName), "bad name"); - LOG("<- {}", typeCtx->forceFind(".toplevel")->type); auto generics = std::vector{ typeCtx->forceFind(".toplevel")->type, std::make_shared(this, rev(f.ast->name))}; @@ -530,6 +531,10 @@ void Cache::populatePythonModule() { } } + // Handle pending realizations! + auto pr = pendingRealizations; // copy it as it might be modified + for (auto &fn : pr) + TranslateVisitor(codegenCtx).transform(functions[fn.first].ast->clone()); typeCtx->age = oldAge; } diff --git a/codon/parser/visitors/typecheck/call.cpp b/codon/parser/visitors/typecheck/call.cpp index f85fa5eb..f82f0c6c 100644 --- a/codon/parser/visitors/typecheck/call.cpp +++ b/codon/parser/visitors/typecheck/call.cpp @@ -840,7 +840,7 @@ ExprPtr TypecheckVisitor::transformSetAttr(CallExpr *expr) { return transform(N(N(expr->args[0].value, staticTyp->evaluate().getString(), expr->args[1].value), - N())); + N(N("NoneType")))); } /// Raise a compiler error. @@ -872,6 +872,7 @@ ExprPtr TypecheckVisitor::transformTupleFn(CallExpr *expr) { ExprPtr TypecheckVisitor::transformTypeFn(CallExpr *expr) { expr->markType(); transform(expr->args[0].value); + unify(expr->type, expr->args[0].value->getType()); if (!realize(expr->type)) @@ -961,9 +962,13 @@ ExprPtr TypecheckVisitor::transformInternalStaticFn(CallExpr *expr) { if (!fn) error("expected a function, got '{}'", expr->args[0].value->type->prettyString()); std::vector v; - for (size_t i = 0; i < fn->ast->args.size(); i++) + for (size_t i = 0; i < fn->ast->args.size(); i++) { + auto n = fn->ast->args[i].name; + trimStars(n); + n = ctx->cache->rev(n); v.push_back(N(std::vector{ - N(i), N(ctx->cache->rev(fn->ast->args[i].name))})); + N(i), N(n)})); + } return transform(N(v)); } else { return nullptr; diff --git a/codon/parser/visitors/typecheck/loops.cpp b/codon/parser/visitors/typecheck/loops.cpp index 8063b682..3e505b2f 100644 --- a/codon/parser/visitors/typecheck/loops.cpp +++ b/codon/parser/visitors/typecheck/loops.cpp @@ -224,7 +224,9 @@ StmtPtr TypecheckVisitor::transformStaticForLoop(ForStmt *stmt) { auto name = ctx->getStaticString(generics[1]); seqassert(name, "bad static string"); if (auto n = in(ctx->cache->classes[typ->name].methods, *name)) { - for (auto &method : ctx->cache->overloads[*n]) { + auto &mt = ctx->cache->overloads[*n]; + for (int mti = int(mt.size()) - 1; mti >= 0; mti--) { + auto &method = mt[mti]; if (endswith(method.name, ":dispatch") || !ctx->cache->functions[method.name].type) continue; diff --git a/stdlib/internal/python.codon b/stdlib/internal/python.codon index b7b4ce9f..f4fb4975 100644 --- a/stdlib/internal/python.codon +++ b/stdlib/internal/python.codon @@ -1517,6 +1517,8 @@ def _____(): __pyenv__ # make it global! import internal.static as _S class _PyWrap: + def _wrap_arg(arg: cobj): + return pyobj(arg, steal=True) def _wrap(args, T: type, F: Static[str], map): for fn in _S.fn_overloads(T, F): if _S.fn_can_call(fn, *args): @@ -1527,8 +1529,9 @@ class _PyWrap: raise PyError("cannot dispatch " + F) def _wrap_unary(obj: cobj, T: type, F: Static[str]) -> cobj: + # print(f'[c] unary: {T.__class__.__name__} {F}') return _PyWrap._wrap( - (pyobj(obj), ), T=T, F=F, + (_PyWrap._wrap_arg(obj), ), T=T, F=F, map=lambda f, a: f(*a).__to_py__() ) def wrap_magic_abs(obj: cobj, T: type): @@ -1552,7 +1555,7 @@ class _PyWrap: def _wrap_hash(obj: cobj, T: type, F: Static[str]) -> i64: return _PyWrap._wrap( - (pyobj(obj), ), T=T, F=F, + (_PyWrap._wrap_arg(obj), ), T=T, F=F, map=lambda f, a: f(*a) ) def wrap_magic_len(obj: cobj, T: type): @@ -1562,34 +1565,42 @@ class _PyWrap: def wrap_magic_bool(obj: cobj, T: type) -> i32: return _PyWrap._wrap( - (pyobj(obj), ), T=T, F="__bool__", + (_PyWrap._wrap_arg(obj), ), T=T, F="__bool__", map=lambda f, a: i32(f(*a)) ) def wrap_magic_del(obj: cobj, T: type): _PyWrap._wrap( - (pyobj(obj), ), T=T, F="__del__", + (_PyWrap._wrap_arg(obj), ), T=T, F="__del__", map=lambda f, a: f(*a) ) def wrap_magic_contains(obj: cobj, arg: cobj, T: type) -> i32: return _PyWrap._wrap( - (pyobj(obj), pyobj(arg)), T=T, F="__contains__", + (_PyWrap._wrap_arg(obj), _PyWrap._wrap_arg(arg)), T=T, F="__contains__", map=lambda f, a: i32(f(*a)) ) def wrap_magic_init(obj: cobj, _args: cobj, _kwds: cobj, T: type) -> i32: - args = pyobj(_args) - kwds = pyobj(_kwds) + # print(f'[c] init: {T.__class__.__name__}') + + args = _PyWrap._wrap_arg(_args) + kwds = _PyWrap._wrap_arg(_kwds) if _kwds != cobj() else None + + # print(f'[c] args: {args}') + # print(f'[c] kwargs: {kwds}') + for fn in _S.fn_overloads(T, "__init__"): try: ai = -1 - # TODO: default values do not work + # TODO: default values do not work; same for *args/**kwargs a = tuple( - kwds[n] if n in kwds else args[(ai := ai + 1)] - for _, n in _S.fn_args(fn) + _PyWrap._wrap_arg(obj) if i == 0 else + (kwds[n] if kwds and n in kwds else args[(ai := ai + 1)]) + for i, n in _S.fn_args(fn) ) - a = (pyobj(obj), *a) + if ai + 1 != args.__len__(): + continue if _S.fn_can_call(fn, *a): fn(*a) return i32(0) @@ -1598,16 +1609,19 @@ class _PyWrap: return i32(-1) def wrap_magic_call(obj: cobj, _args: cobj, _kwds: cobj, T: type) -> cobj: - args = pyobj(_args) - kwds = pyobj(_kwds) + args = _PyWrap._wrap_arg(_args) + kwds = _PyWrap._wrap_arg(_kwds) if _kwds != cobj() else None for fn in _S.fn_overloads(T, "__call__"): try: ai = -1 - a = tuple( # TODO: default values do not work - kwds[n] if n in kwds else args[(ai := ai + 1)] - for _, n in _S.fn_args(fn) + # TODO: default values do not work; same for *args/**kwargs + a = tuple( + _PyWrap._wrap_arg(obj) if i == 0 else + (kwds[n] if kwds and n in kwds else args[(ai := ai + 1)]) + for i, n in _S.fn_args(fn) ) - a = (pyobj(obj), *a) + if ai + 1 != args.__len__(): + continue if _S.fn_can_call(fn, *a): return fn(*a).__to_py__() except PyError: @@ -1616,7 +1630,7 @@ class _PyWrap: def _wrap_cmp(obj: cobj, other: cobj, T: type, F: Static[str]) -> cobj: return _PyWrap._wrap( - (pyobj(obj), pyobj(other)), T=T, F=F, + (_PyWrap._wrap_arg(obj), _PyWrap._wrap_arg(other)), T=T, F=F, map=lambda f, a: f(*a).__to_py__() ) def wrap_magic_lt(obj: cobj, other: cobj, T: type): @@ -1652,14 +1666,14 @@ class _PyWrap: if val == cobj(): try: if hasattr(T, "__delitem__"): - T.__delitem__(pyobj(obj), pyobj(idx)) + T.__delitem__(_PyWrap._wrap_arg(obj), _PyWrap._wrap_arg(idx)) return 0 except PyError: pass return -1 try: _PyWrap._wrap( - (pyobj(obj), pyobj(idx), pyobj(val)), T=T, F="__setitem__", + (_PyWrap._wrap_arg(obj), _PyWrap._wrap_arg(idx), _PyWrap._wrap_arg(val)), T=T, F="__setitem__", map=lambda f, a: f(*a).__to_py__() ) return 0 @@ -1696,32 +1710,46 @@ class _PyWrap: return _PyWrap.wrap_from_py(obj, _PyWrap.IterWrap[T]) def wrap_magic_iter(obj: cobj, T: type) -> cobj: + # print('[c] iter') return _PyWrap.IterWrap._init(obj, T) - def wrap_single(obj: cobj, arg: cobj, T: type, F: Static[str], method: Static[int]): - a = (pyobj(obj), pyobj(arg)) if method else (pyobj(arg),) - return _PyWrap._wrap( - a, T=T, F=F, - map=lambda f, a: f(*a).__to_py__() - ) - - def wrap_multiple(obj: cobj, args: Ptr[cobj], nargs: i32, T: type, F: Static[str], method: Static[int]): - def _err(): + def wrap_multiple_method(obj: cobj, args: Ptr[cobj], nargs: int, T: type, F: Static[str]): + # print(f'[c] method: {T.__class__.__name__} {F} {obj} {args} {nargs}') + def _err() -> pyobj: raise PyError("argument mismatch") - return pyobj() - a = (pyobj(obj), ) if method else () for fn in _S.fn_overloads(T, F): try: ai = -1 - an = ( - pyobj(args[i]) if i < nargs else _err() + an = tuple( + _PyWrap._wrap_arg(obj) if i == 0 else + (_PyWrap._wrap_arg(args[i]) if i < nargs else _err()) + for i, _ in _S.fn_args(fn) + ) + if len(an) != nargs + 1: + _err() + if _S.fn_can_call(fn, *an): + return fn(*an).__to_py__() + except PyError: + pass + PyError("cannot dispatch " + F) + + def wrap_multiple(obj: cobj, args: Ptr[cobj], nargs: int, T: type, F: Static[str]): + # print(f'[c] nonmethod: {T.__class__.__name__} {F} {obj} {args} {nargs}') + def _err() -> pyobj: + raise PyError("argument mismatch") + + for fn in _S.fn_overloads(T, F): + try: + ai = -1 + an = tuple( + _PyWrap._wrap_arg(args[i]) if i < nargs else _err() for i, _ in _S.fn_args(fn) ) if len(an) != nargs: _err() - if _S.fn_can_call(fn, (*a, *an)): - return fn(*a, *an).__to_py__() + if _S.fn_can_call(fn, *an): + return fn(*an).__to_py__() except PyError: pass PyError("cannot dispatch " + F) @@ -1731,7 +1759,8 @@ class _PyWrap: def wrap_set(obj: cobj, what: cobj, closure: cobj, T: type, S: Static[str]) -> i32: try: t = T.__from_py__(obj) - setattr(t, S, type(getattr(t, S)).__from_py__(what)) + val = type(getattr(t, S)).__from_py__(what) + setattr(t, S, val) return i32(0) except PyError: return i32(-1)