Fix miscellaneous issues (#85)

* Fix #75

* Build on macOS 11 for #72

* Fix #76

* Fix test

* Fix generator example in docs

* Fix Python/Cython packaging; Fix #80

* Fix tests

* Fix tests

* Fix tests

* Fix tests

* Fix syntax

* Fix CI

* Fix CI

* Fix CI; Dominate imported bindings

* Fix CI; Dominate imported bindings

* Fix .gitignore

* Fix tests

* Fix rm command

* Format

* clang-format

* Organize and optimize string formatting

* Fix format error message

Co-authored-by: Ibrahim Numanagić <ibrahimpasa@gmail.com>
pull/117/head
A. R. Shajii 2022-12-12 20:54:01 -05:00 committed by GitHub
parent f4feee23a4
commit f4323a80a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 504 additions and 258 deletions

View File

@ -9,7 +9,7 @@ yum -y install python3 python3-devel
# env
export PYTHONPATH=$(pwd)/test/python
export CODON_PYTHON=$(python3 test/python/find-python-library.py)
python3 -m pip install --upgrade pip
python3 -m pip install -Iv pip==21.3.1
python3 -m pip install numpy
# deps
@ -26,31 +26,24 @@ export LLVM_DIR=$(llvm/bin/llvm-config --cmakedir)
-DCMAKE_C_COMPILER=${CC} \
-DCMAKE_CXX_COMPILER=${CXX})
cmake --build build --config Release -- VERBOSE=1
cmake --install build --prefix=codon-deploy
# build cython
export PATH=$PATH:$(pwd)/llvm/bin
export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH
export CODON_DIR=$(pwd)/build
python3 -m pip install cython wheel astunparse
python3 -m pip debug --verbose
(cd extra/python; python3 setup.py sdist bdist_wheel --plat-name=manylinux2014_x86_64)
python3 -m pip install -v extra/python/dist/*.whl
export PYTHONPATH=$(pwd):$PYTHONPATH
(cd codon-deploy/python && python3 setup.py sdist)
CODON_DIR=$(pwd)/codon-deploy python3 -m pip install -v codon-deploy/python/dist/*.gz
python3 test/python/cython_jit.py
# test
export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH
export PYTHONPATH=$(pwd):$PYTHONPATH
export CODON_PATH=$(pwd)/stdlib
ln -s build/libcodonrt.so .
build/codon_test
# package
export CODON_BUILD_ARCHIVE=codon-$(uname -s | awk '{print tolower($0)}')-$(uname -m).tar.gz
mkdir -p codon-deploy/bin codon-deploy/lib/codon codon-deploy/plugins
cp build/codon codon-deploy/bin/
cp build/libcodon*.so codon-deploy/lib/codon/
cp build/libomp.so codon-deploy/lib/codon/
cp -r build/include codon-deploy/
cp -r stdlib codon-deploy/lib/codon/
cp -r extra/python/dist/*.whl codon-deploy/
rm -rf codon-deploy/lib/libfmt.a codon-deploy/lib/pkgconfig codon-deploy/lib/cmake codon-deploy/python/codon.egg-info codon-deploy/python/dist codon-deploy/python/build
tar -czf ${CODON_BUILD_ARCHIVE} codon-deploy
du -sh codon-deploy

View File

@ -100,7 +100,7 @@ jobs:
matrix:
os:
- ubuntu-latest
- macos-latest
- macos-11
runs-on: ${{ matrix.os }}
name: Codon CI
needs: [ release ]
@ -153,24 +153,15 @@ jobs:
-DCMAKE_C_COMPILER=${CC} \
-DCMAKE_CXX_COMPILER=${CXX})
cmake --build build --config Release -- VERBOSE=1
cmake --install build --prefix=codon-deploy
env:
CC: clang
CXX: clang++
- name: Build Cython
if: startsWith(matrix.os, 'ubuntu')
run: |
(cd extra/python; CODON_DIR=../../build python3 setup.py sdist bdist_wheel)
python -m pip install -v extra/python/dist/*.whl
env:
CC: clang
CXX: clang++
- name: Build Cython
if: startsWith(matrix.os, 'macos')
run: |
(cd extra/python; CODON_DIR=../../build python3 setup.py sdist bdist_wheel --plat-name=macosx_10_16_x86_64)
python -m pip install -v extra/python/dist/*.whl
(cd codon-deploy/python && python3 setup.py sdist)
CODON_DIR=$(pwd)/codon-deploy python -m pip install -v codon-deploy/python/dist/*.gz
python test/python/cython_jit.py
env:
CC: clang
@ -192,14 +183,9 @@ jobs:
- name: Prepare Artifacts
run: |
mkdir -p codon-deploy/bin codon-deploy/lib/codon codon-deploy/plugins
cp build/codon codon-deploy/bin/
cp build/libcodon*.${LIBEXT} codon-deploy/lib/codon/
cp build/libomp.${LIBEXT} codon-deploy/lib/codon/
cp -rf codon-deploy/python/dist .
rm -rf codon-deploy/lib/libfmt.a codon-deploy/lib/pkgconfig codon-deploy/lib/cmake codon-deploy/python/codon.egg-info codon-deploy/python/dist codon-deploy/python/build
if [ -e build/libunwind.${LIBEXT} ]; then cp build/libunwind.${LIBEXT} codon-deploy/lib/codon/; fi
cp -r build/include codon-deploy/
cp -r stdlib codon-deploy/lib/codon/
cp -r extra/python/dist/*.whl codon-deploy/
tar -czf ${CODON_BUILD_ARCHIVE} codon-deploy
du -sh codon-deploy
@ -244,3 +230,20 @@ jobs:
with:
name: ${{ matrix.os }}-x86_64
path: codon-linux-x86_64.tar.gz
- name: Publish on TestPyPI
if: startsWith(matrix.os, 'ubuntu')
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_TEST_TOKEN }}
repository_url: https://test.pypi.org/legacy/
skip_existing: true
- name: Publish Package
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && startsWith(matrix.os, 'ubuntu')
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: ${{ secrets.PYPI_USERNAME }}
password: ${{ secrets.PYPI_PASSWORD }}

4
.gitignore vendored
View File

@ -16,6 +16,7 @@
*.pyc
build/
build_*/
install/
extra/python/src/jit.cpp
extra/jupyter/build/
@ -55,7 +56,8 @@ Thumbs.db
.vscode
extra/jupyter/share/jupyter/kernels/codon/kernel.json
scratch.*
extra/python/codon/version.py
scratch*.*
_*
.ipynb_checkpoints

View File

@ -7,7 +7,7 @@ project(
configure_file("${PROJECT_SOURCE_DIR}/cmake/config.h.in"
"${PROJECT_SOURCE_DIR}/codon/config/config.h")
configure_file("${PROJECT_SOURCE_DIR}/cmake/config.py.in"
"${PROJECT_SOURCE_DIR}/extra/python/config/config.py")
"${PROJECT_SOURCE_DIR}/extra/python/codon/version.py")
option(CODON_JUPYTER "build Codon Jupyter server" OFF)
option(CODON_GPU "build Codon GPU backend" OFF)
@ -431,14 +431,10 @@ FetchContent_Declare(
URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
)
# For Windows: Prevent overriding the parent project's compiler/linker settings
set(gtest_force_shared_crt
ON
CACHE BOOL "" FORCE)
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
option(INSTALL_GTEST "Enable installation of googletest." OFF)
FetchContent_MakeAvailable(googletest)
enable_testing()
if(IS_DIRECTORY "${googletest_SOURCE_DIR}")
set_property(DIRECTORY ${googletest_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL YES)
endif()
set(CODON_TEST_CPPFILES
test/main.cpp
test/sir/analyze/dominator.cpp
@ -467,4 +463,5 @@ install(FILES ${CMAKE_BINARY_DIR}/libomp${CMAKE_SHARED_LIBRARY_SUFFIX} DESTINATI
install(TARGETS codon DESTINATION bin)
install(DIRECTORY ${CMAKE_BINARY_DIR}/include/codon DESTINATION include)
install(DIRECTORY ${CMAKE_SOURCE_DIR}/stdlib DESTINATION lib/codon)
install(DIRECTORY ${CMAKE_SOURCE_DIR}/extra/python/ DESTINATION python)
install(DIRECTORY DESTINATION lib/codon/plugins)

View File

@ -1,3 +1,5 @@
__version__ = "0.1"
CODON_VERSION = "@PROJECT_VERSION@"
CODON_VERSION_MAJOR = @PROJECT_VERSION_MAJOR@
CODON_VERSION_MINOR = @PROJECT_VERSION_MINOR@

View File

@ -472,7 +472,7 @@ template <class... TA> std::string Emsg(Error e, const TA &...args) {
args...);
case Error::COMPILER_NO_FILE:
return fmt::format("cannot open file '{}' for parsing");
return fmt::format("cannot open file '{}' for parsing", args...);
case Error::COMPILER_NO_STDLIB:
return fmt::format("cannot locate standard library");
case Error::MAX_REALIZATION:

View File

@ -4,6 +4,7 @@
#include <sstream>
#include "codon/parser/common.h"
#include "codon/parser/peg/peg.h"
#include "codon/parser/visitors/doc/doc.h"
#include "codon/parser/visitors/format/format.h"
@ -345,5 +346,7 @@ JITResult jitExecuteSafe(JIT *jit, const std::string &code, const std::string &f
return jit->executeSafe(code, file, line, debug);
}
std::string getJITLibrary() { return ast::library_path(); }
} // namespace jit
} // namespace codon

View File

@ -2,6 +2,7 @@
#include "common.h"
#include <inttypes.h>
#include <string>
#include <vector>
@ -146,6 +147,64 @@ std::string executable_path(const char *argv0) {
return llvm::sys::fs::getMainExecutable(argv0, p);
}
// Adopted from https://github.com/gpakosz/whereami/blob/master/src/whereami.c (MIT)
#ifdef __APPLE__
#include <dlfcn.h>
#include <mach-o/dyld.h>
#endif
std::string library_path() {
std::string result;
#ifdef __APPLE__
char buffer[PATH_MAX];
for (;;) {
Dl_info info;
if (dladdr(__builtin_extract_return_addr(__builtin_return_address(0)), &info)) {
char *resolved = realpath(info.dli_fname, buffer);
if (!resolved)
break;
result = std::string(resolved);
}
break;
}
#else
for (int r = 0; r < 5; r++) {
FILE *maps = fopen("/proc/self/maps", "r");
if (!maps)
break;
for (;;) {
char buffer[PATH_MAX < 1024 ? 1024 : PATH_MAX];
uint64_t low, high;
char perms[5];
uint64_t offset;
uint32_t major, minor;
char path[PATH_MAX];
uint32_t inode;
if (!fgets(buffer, sizeof(buffer), maps))
break;
if (sscanf(buffer, "%" PRIx64 "-%" PRIx64 " %s %" PRIx64 " %x:%x %u %s\n", &low,
&high, perms, &offset, &major, &minor, &inode, path) == 8) {
uint64_t addr =
(uintptr_t)(__builtin_extract_return_addr(__builtin_return_address(0)));
if (low <= addr && addr <= high) {
char *resolved = realpath(path, buffer);
if (resolved)
result = std::string(resolved);
break;
}
}
}
fclose(maps);
if (!result.empty())
break;
}
#endif
return result;
}
namespace {
void addPath(std::vector<std::string> &paths, const std::string &path) {

View File

@ -139,9 +139,12 @@ template <typename T> std::vector<T> clone_nop(const std::vector<T> &t) {
/// @return The absolute canonical path of a given path.
std::string getAbsolutePath(const std::string &path);
/// Detect a absolute path of the current executable (whose argv0 is known).
/// Detect an absolute path of the current executable (whose argv0 is known).
/// @return Absolute executable path or argv0 if one cannot be found.
std::string executable_path(const char *argv0);
/// Detect an absolute path of the current libcodonc.
/// @return Absolute executable path or argv0 if one cannot be found.
std::string library_path();
struct ImportFile {
enum Status { STDLIB, PACKAGE };

View File

@ -84,13 +84,16 @@ void SimplifyVisitor::visit(ImportStmt *stmt) {
seqassert(stmt->as.empty(), "renamed star-import");
// Just copy all symbols from import's context here.
for (auto &i : *(import.ctx)) {
if (!i.second.front()->isConditional() &&
(!startswith(i.first, "_") ||
if ((!startswith(i.first, "_") ||
(ctx->isStdlibLoading && startswith(i.first, "__")))) {
// Ignore all identifiers that start with `_` but not those that start with
// `__` while the standard library is being loaded
/// TODO: handle conditionals as well?
ctx->add(i.first, i.second.front());
auto c = i.second.front();
if (c->isConditional() && i.first.find('.') == std::string::npos) {
LOG("-> fix {} :: {}", import.moduleName, i.first);
c = import.ctx->findDominatingBinding(i.first);
}
ctx->add(i.first, c);
}
}
} else {
@ -99,8 +102,10 @@ void SimplifyVisitor::visit(ImportStmt *stmt) {
seqassert(i, "not a valid import what expression");
auto c = import.ctx->find(i->value);
// Make sure that we are importing an existing global symbol
if (!c || c->isConditional())
if (!c)
E(Error::IMPORT_NO_NAME, i, i->value, file->module);
if (c->isConditional())
c = import.ctx->findDominatingBinding(i->value);
ctx->add(stmt->as.empty() ? i->value : stmt->as, c);
}

View File

@ -161,9 +161,19 @@ types::TypePtr TypecheckVisitor::realize(types::TypePtr typ) {
? f->funcGenerics[gi++].type->prettyString()
: f->getArgTypes()[ai++]->prettyString()));
}
e.trackRealize(
fmt::format("{}({})", ctx->cache->rev(f->ast->name), fmt::join(args, ", ")),
getSrcInfo());
auto name = f->ast->name;
std::string name_args;
if (startswith(name, "._import_")) {
name = name.substr(10);
auto p = name.find('_');
if (p != std::string::npos)
name = name.substr(0, p);
name = "<import " + name + ">";
} else {
name = ctx->cache->rev(f->ast->name);
name_args = fmt::format("({})", fmt::join(args, ", "));
}
e.trackRealize(fmt::format("{}{}", name, name_args), getSrcInfo());
}
} else {

View File

@ -242,42 +242,49 @@ static seq_str_t string_conv(const std::string &s) {
return {(seq_int_t)s.size(), p};
}
template <typename T>
static seq_str_t fmt_conv(T n, const std::string &default_fmt, char *format,
char *error) {
std::string ret;
template <typename T> std::string default_format(T n) {
return fmt::format(FMT_STRING("{}"), n);
}
template <> std::string default_format(double n) {
return fmt::format(FMT_STRING("{:g}"), n);
}
template <typename T> seq_str_t fmt_conv(T n, seq_str_t format, bool *error) {
*error = false;
try {
*error = false;
if (!format) {
ret = fmt::format(default_fmt, n);
if (format.len == 0) {
return string_conv(default_format(n));
} else {
ret = fmt::format(fmt::runtime(fmt::format("{{:{}}}", format)), n);
std::string fstr(format.str, format.len);
return string_conv(
fmt::format(fmt::runtime(fmt::format(FMT_STRING("{{:{}}}"), fstr)), n));
}
} catch (const std::runtime_error &f) {
*error = true;
ret = f.what();
return string_conv(f.what());
}
return string_conv(ret);
}
SEQ_FUNC seq_str_t seq_str_int(seq_int_t n, char *format, char *error) {
return fmt_conv<seq_int_t>(n, "{}", format, error);
SEQ_FUNC seq_str_t seq_str_int(seq_int_t n, seq_str_t format, bool *error) {
return fmt_conv<seq_int_t>(n, format, error);
}
SEQ_FUNC seq_str_t seq_str_uint(seq_int_t n, char *format, char *error) {
return fmt_conv<uint64_t>(n, "{}", format, error);
SEQ_FUNC seq_str_t seq_str_uint(seq_int_t n, seq_str_t format, bool *error) {
return fmt_conv<uint64_t>(n, format, error);
}
SEQ_FUNC seq_str_t seq_str_float(double f, char *format, char *error) {
return fmt_conv<double>(f, "{:g}", format, error);
SEQ_FUNC seq_str_t seq_str_float(double f, seq_str_t format, bool *error) {
return fmt_conv<double>(f, format, error);
}
SEQ_FUNC seq_str_t seq_str_ptr(void *p, char *format, char *error) {
return fmt_conv(fmt::ptr(p), "{}", format, error);
SEQ_FUNC seq_str_t seq_str_ptr(void *p, seq_str_t format, bool *error) {
return fmt_conv(fmt::ptr(p), format, error);
}
SEQ_FUNC seq_str_t seq_str_str(char *p, char *format, char *error) {
return fmt_conv(p, "{}", format, error);
SEQ_FUNC seq_str_t seq_str_str(seq_str_t s, seq_str_t format, bool *error) {
std::string t(s.str, s.len);
return fmt_conv(t, format, error);
}
/*

View File

@ -76,12 +76,11 @@ SEQ_FUNC _Unwind_Reason_Code seq_personality(int version, _Unwind_Action actions
SEQ_FUNC int64_t seq_exc_offset();
SEQ_FUNC uint64_t seq_exc_class();
SEQ_FUNC seq_str_t seq_str_int(seq_int_t n, char *, char *);
SEQ_FUNC seq_str_t seq_str_uint(seq_int_t n, char *, char *);
SEQ_FUNC seq_str_t seq_str_float(double f, char *, char *);
SEQ_FUNC seq_str_t seq_str_ptr(void *p, char *, char *);
SEQ_FUNC seq_str_t seq_str_str(char *p, char *, char *);
SEQ_FUNC seq_str_t seq_str_tuple(seq_str_t *strs, seq_int_t n);
SEQ_FUNC seq_str_t seq_str_int(seq_int_t n, seq_str_t format, bool *error);
SEQ_FUNC seq_str_t seq_str_uint(seq_int_t n, seq_str_t format, bool *error);
SEQ_FUNC seq_str_t seq_str_float(double f, seq_str_t format, bool *error);
SEQ_FUNC seq_str_t seq_str_ptr(void *p, seq_str_t format, bool *error);
SEQ_FUNC seq_str_t seq_str_str(seq_str_t s, seq_str_t format, bool *error);
SEQ_FUNC void *seq_stdin();
SEQ_FUNC void *seq_stdout();

View File

@ -6,7 +6,8 @@ pow(2.0, 2.0) # 4.0
# Import and rename function
# cobj is a C pointer (void*, char*, etc.)
from C import puts(cobj) -> void as print_line
# None can be used to represent C's void
from C import puts(cobj) -> None as print_line
print_line("hello".c_str()) # prints "hello"; c_str() converts Codon str to C string
```

View File

@ -2,8 +2,9 @@ Codon supports generators, and in fact they are heavily optimized in
the compiler so as to typically eliminate any overhead:
``` python
def gen(i):
while i < 10:
def gen(n):
i = 0
while i < n:
yield i ** 2
i += 1

View File

@ -1,3 +1 @@
recursive-include codon/stdlib *.codon
recursive-include codon *.so
recursive-include codon *.dylib
include codon/jit.h

View File

@ -1,9 +1,11 @@
To install:
```bash
$ pip install extra/python
$ pip install .
```
If Codon is installed in non-standard directory, please set `CODON_DIR` accordingly.
To use:
```python

View File

@ -8,29 +8,37 @@ import os
import functools
import itertools
import ast
import shutil
import astunparse
from pathlib import Path
sys.setdlopenflags(sys.getdlopenflags() | ctypes.RTLD_GLOBAL)
from .codon_jit import JITWrapper, JITError, codon_library
if "CODON_PATH" not in os.environ:
os.environ["CODON_PATH"] = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))
)
os.environ["CODON_PATH"] += "/stdlib"
codon_lib_path = codon_library()
if not codon_lib_path:
raise RuntimeError(
"Cannot locate Codon. Please install Codon or set CODON_PATH."
)
codon_path = (Path(codon_lib_path).parent / "stdlib").resolve()
os.environ["CODON_PATH"] = str(codon_path)
from .codon_jit import JITWrapper, JITError
pod_conversions = {type(None): "pyobj",
int: "int",
float: "float",
bool: "bool",
str: "str",
complex: "complex",
slice: "slice"}
pod_conversions = {
type(None): "pyobj",
int: "int",
float: "float",
bool: "bool",
str: "str",
complex: "complex",
slice: "slice",
}
custom_conversions = {}
_error_msgs = set()
def _common_type(t, debug, sample_size):
sub, is_optional = None, False
for i in itertools.islice(t, sample_size):
@ -42,9 +50,10 @@ def _common_type(t, debug, sample_size):
return "pyobj"
sub = s
if is_optional and sub and sub != "pyobj":
sub = f"Optional[{sub}]"
sub = "Optional[{}]".format(sub)
return sub if sub else "pyobj"
def _codon_type(arg, **kwargs):
t = type(arg)
@ -52,40 +61,48 @@ def _codon_type(arg, **kwargs):
if s:
return s
if issubclass(t, list):
return f"List[{_common_type(arg, **kwargs)}]"
return "List[{}]".format(_common_type(arg, **kwargs))
if issubclass(t, set):
return f"Set[{_common_type(arg, **kwargs)}]"
return "Set[{}]".format(_common_type(arg, **kwargs))
if issubclass(t, dict):
return f"Dict[{_common_type(arg.keys(), **kwargs)},{_common_type(arg.values(), **kwargs)}]"
return "Dict[{},{}]".format(
_common_type(arg.keys(), **kwargs), _common_type(arg.values(), **kwargs)
)
if issubclass(t, tuple):
return f"Tuple[{','.join(_codon_type(a, **kwargs) for a in arg)}]"
return "Tuple[{}]".format(",".join(_codon_type(a, **kwargs) for a in arg))
s = custom_conversions.get(t, "")
if s:
j = ','.join(_codon_type(getattr(arg, slot), **kwargs) for slot in t.__slots__)
return f"{s}[{j}]"
j = ",".join(_codon_type(getattr(arg, slot), **kwargs) for slot in t.__slots__)
return "{}[{}]".format(s, j)
debug = kwargs.get('debug', None)
debug = kwargs.get("debug", None)
if debug:
msg = f"cannot convert {t.__name__}"
msg = "cannot convert " + t.__name__
if msg not in _error_msgs:
print(f"[python] {msg}", file=sys.stderr)
print("[python]", msg, file=sys.stderr)
_error_msgs.add(msg)
return "pyobj"
def _codon_types(args, **kwargs):
return tuple(_codon_type(arg, **kwargs) for arg in args)
def _reset_jit():
global _jit
_jit = JITWrapper()
init_code = ("from internal.python import "
"setup_decorator, PyTuple_GetItem, PyObject_GetAttrString\n"
"setup_decorator()\n")
init_code = (
"from internal.python import "
"setup_decorator, PyTuple_GetItem, PyObject_GetAttrString\n"
"setup_decorator()\n"
)
_jit.execute(init_code, "", 0, False)
return _jit
_jit = _reset_jit()
class RewriteFunctionArgs(ast.NodeTransformer):
def __init__(self, args):
self.args = args
@ -95,6 +112,7 @@ class RewriteFunctionArgs(ast.NodeTransformer):
node.args.args.append(ast.arg(arg=a, annotation=None))
return node
def _obj_to_str(obj, **kwargs) -> str:
if inspect.isclass(obj):
lines = inspect.getsourcelines(obj)[0]
@ -104,50 +122,65 @@ def _obj_to_str(obj, **kwargs) -> str:
lines = inspect.getsourcelines(obj)[0]
extra_spaces = lines[0].find("@")
obj_str = "".join(l[extra_spaces:] for l in lines[1:])
if kwargs.get('pyvars', None):
if kwargs.get("pyvars", None):
node = ast.fix_missing_locations(
RewriteFunctionArgs(kwargs['pyvars']).visit(ast.parse(obj_str))
RewriteFunctionArgs(kwargs["pyvars"]).visit(ast.parse(obj_str))
)
obj_str = astunparse.unparse(node)
else:
raise TypeError(f"Function or class expected, got {type(obj).__name__}.")
raise TypeError("Function or class expected, got " + type(obj).__name__)
return obj_str.replace("_@par", "@par")
def _obj_name(obj) -> str:
if inspect.isclass(obj) or callable(obj):
return obj.__name__
else:
raise TypeError(f"Function or class expected, got {type(obj).__name__}.")
raise TypeError("Function or class expected, got " + type(obj).__name__)
def _parse_decorated(obj, **kwargs):
return _obj_name(obj), _obj_to_str(obj, **kwargs)
def convert(t):
if not hasattr(t, "__slots__"):
raise JITError(f"class '{str(t)}' does not have '__slots__' attribute")
raise JITError("class '{}' does not have '__slots__' attribute".format(str(t)))
name = t.__name__
slots = t.__slots__
code = ("@tuple\n"
"class " + name + "[" + ",".join(f"T{i}" for i in range(len(slots))) + "]:\n")
code = (
"@tuple\n"
"class "
+ name
+ "["
+ ",".join("T{}".format(i) for i in range(len(slots)))
+ "]:\n"
)
for i, slot in enumerate(slots):
code += f" {slot}: T{i}\n"
code += " {}: T{}\n".format(slot, i)
# PyObject_GetAttrString
code += " def __from_py__(p: cobj):\n"
for i, slot in enumerate(slots):
code += f" a{i} = T{i}.__from_py__(PyObject_GetAttrString(p, '{slot}'.ptr))\n"
code += f" return {name}({', '.join(f'a{i}' for i in range(len(slots)))})\n"
code += " a{} = T{}.__from_py__(PyObject_GetAttrString(p, '{}'.ptr))\n".format(
i, i, slot
)
code += " return {}({})\n".format(
name, ", ".join("a{}".format(i) for i in range(len(slots)))
)
_jit.execute(code, "", 0, False)
custom_conversions[t] = name
return t
def jit(fn=None, debug=None, sample_size=5, pyvars=None):
if not pyvars:
pyvars = []
if not isinstance(pyvars, list):
raise ArgumentError("pyvars must be a list")
def _decorate(f):
try:
obj_name, obj_str = _parse_decorated(f, pyvars=pyvars)
@ -155,7 +188,7 @@ def jit(fn=None, debug=None, sample_size=5, pyvars=None):
obj_str,
f.__code__.co_filename,
f.__code__.co_firstlineno,
1 if debug else 0
1 if debug else 0,
)
except JITError:
_reset_jit()
@ -167,12 +200,19 @@ def jit(fn=None, debug=None, sample_size=5, pyvars=None):
args = (*args, *kwargs.values())
types = _codon_types(args, debug=debug, sample_size=sample_size)
if debug:
print(f"[python] {f.__name__}({list(types)})", file=sys.stderr)
return _jit.run_wrapper(obj_name, types, f.__module__, pyvars, args, 1 if debug else 0)
print(
"[python] {}({})".format(f.__name__, list(types)),
file=sys.stderr,
)
return _jit.run_wrapper(
obj_name, types, f.__module__, pyvars, args, 1 if debug else 0
)
except JITError:
_reset_jit()
raise
return wrapped
if fn:
return _decorate(fn)
return _decorate

View File

@ -30,5 +30,7 @@ JITResult jitExecutePython(JIT *jit, const std::string &name,
JITResult jitExecuteSafe(JIT *jit, const std::string &code, const std::string &file,
int line, bool debug);
std::string getJITLibrary();
} // namespace jit
} // namespace codon

View File

@ -13,3 +13,4 @@ cdef extern from "jit.h" namespace "codon::jit":
JIT *jitInit(string)
JITResult jitExecuteSafe(JIT*, string, string, int, char)
JITResult jitExecutePython(JIT*, string, vector[string], string, vector[string], object, char)
string getJITLibrary()

View File

@ -40,3 +40,6 @@ cdef class JITWrapper:
return <object>result.result
else:
raise JITError(result.message)
def codon_library():
return codon.jit.getJITLibrary()

View File

@ -1,3 +0,0 @@
*
*/
!.gitignore

View File

@ -7,52 +7,74 @@ from pathlib import Path
from Cython.Distutils import build_ext
from setuptools import setup
from setuptools.extension import Extension
import tempfile
from config.config import *
codon_dir = Path(os.environ.get(
"CODON_DIR", os.path.realpath(f"{os.getcwd()}/../../build")
))
exec(open("codon/version.py").read())
ext = "dylib" if sys.platform == "darwin" else "so"
root = Path(__file__).resolve().parent
def symlink(target):
tmp = tempfile.mktemp()
os.symlink(str(target.resolve()), tmp)
shutil.move(tmp, str(root / "codon" / target.name))
symlink(codon_dir / ".." / "stdlib")
symlink(codon_dir / "lib" / "codon" / ("libcodonc." + ext))
symlink(codon_dir / "lib" / "codon" / ("libcodonrt." + ext))
symlink(codon_dir / "lib" / "codon" / ("libomp." + ext))
codon_path = os.environ.get("CODON_DIR")
if not codon_path:
c = shutil.which("codon")
if c:
codon_path = Path(c).parent / ".."
else:
codon_path = Path(codon_path)
for path in [
os.path.expanduser("~") + "/.codon",
os.getcwd() + "/..",
]:
path = Path(path)
if not codon_path and path.exists():
codon_path = path
break
if (
not codon_path
or not (codon_path / "include" / "codon").exists()
or not (codon_path / "lib" / "codon").exists()
):
print(
"Cannot find Codon.",
'Please either install Codon (/bin/bash -c "$(curl -fsSL https://exaloop.io/install.sh)"),',
"or set CODON_DIR if Codon is not in PATH or installed in ~/.codon",
file=sys.stderr,
)
sys.exit(1)
codon_path = codon_path.resolve()
print("Codon: " + str(codon_path))
if sys.platform == "darwin":
linker_args = "-Wl,-rpath,@loader_path"
linker_args = "-Wl,-rpath," + str(codon_path / "lib" / "codon")
else:
linker_args = "-Wl,-rpath=$ORIGIN"
linker_args = "-Wl,-rpath=" + str(codon_path / "lib" / "codon")
jit_extension = Extension(
"codon.codon_jit",
sources=["codon/jit.pyx"],
sources=["codon/jit.pyx", "codon/jit.pxd"],
libraries=["codonc", "codonrt"],
language="c++",
extra_compile_args=["-w", "-std=c++17"],
extra_link_args=[linker_args],
library_dirs=[str(root / "codon")],
library_dirs=[str(codon_path / "lib" / "codon")],
)
setup(
name="codon",
version=CODON_VERSION,
install_requires=["astunparse"],
python_requires='>=3.6',
version=__version__,
install_requires=["cython", "astunparse"],
python_requires=">=3.6",
description="Codon JIT decorator",
url="https://exaloop.io",
long_description="Please see https://exaloop.io for more details.",
author="Exaloop Inc.",
author_email="info@exaloop.io",
license="Commercial",
cmdclass={"build_ext": build_ext},
ext_modules=[jit_extension],
packages=["codon"],
include_package_data=True
include_package_data=True,
cmdclass={
"build_ext": build_ext,
},
)

View File

@ -30,6 +30,7 @@ from internal.types.collections.tuple import *
# Extended core library
import internal.c_stubs as _C
from internal.format import *
from internal.builtin import *
from internal.builtin import _jit_display
from internal.str import *

View File

@ -21,6 +21,7 @@ from internal.internal import *
from internal.types.strbuf import strbuf as _strbuf
from internal.types.collections.list import *
import internal.c_stubs as _C
from internal.format import *
def next(g: Generator[T], default: Optional[T] = None, T: type) -> T:
if g.done():
@ -124,6 +125,24 @@ class str:
else:
raise ValueError("nope")
def strip(self):
if self.__len__() == 0:
return ""
i = 0
while i < self.__len__() and _C.isspace(i32(int(self.ptr[i]))):
i += 1
j = self.__len__() - 1
while j >= 0 and _C.isspace(i32(int(self.ptr[j]))):
j -= 1
j += 1
if j <= i:
return ""
return str(self.ptr + i, j - i)
def __repr__(self) -> str:
return f"'{self}'"

View File

@ -1,8 +1,5 @@
# Copyright (C) 2022 Exaloop Inc. <https://exaloop.io>
from internal.gc import alloc_atomic, free
from internal.types.optional import unwrap
class object:
def __init__(self):
pass
@ -317,11 +314,13 @@ def hex(n):
@extend
class int:
def _from_str(s: str, base: int):
from C import strtoll(cobj, Ptr[cobj], i32) -> int
from internal.gc import alloc_atomic, free
if base < 0 or base > 36 or base == 1:
raise ValueError("int() base must be >= 2 and <= 36, or 0")
s0 = s
s = s.strip()
buf = __array__[byte](32)
n = len(s)
need_dyn_alloc = n >= len(buf)
@ -331,18 +330,42 @@ class int:
p[n] = byte(0)
end = cobj()
result = strtoll(p, __ptr__(end), i32(base))
result = _C.strtoll(p, __ptr__(end), i32(base))
if need_dyn_alloc:
free(p)
if end != p + n:
if n == 0 or end != p + n:
raise ValueError(
f"invalid literal for int() with base {base}: {s}"
f"invalid literal for int() with base {base}: {s0.__repr__()}"
)
return result
@extend
class float:
def _from_str(s: str) -> float:
s0 = s
s = s.strip()
buf = __array__[byte](32)
n = len(s)
need_dyn_alloc = n >= len(buf)
p = alloc_atomic(n + 1) if need_dyn_alloc else buf.ptr
str.memcpy(p, s.ptr, n)
p[n] = byte(0)
end = cobj()
result = _C.strtod(p, __ptr__(end))
if need_dyn_alloc:
free(p)
if n == 0 or end != p + n:
raise ValueError(f"could not convert string to float: {s0.__repr__()}")
return result
def _jit_display(x, s: Static[str], bundle: Set[str] = Set[str]()):
if isinstance(x, None):
return

View File

@ -4,6 +4,31 @@
from C import seq_print(str)
from C import seq_print_full(str, cobj)
@nocapture
@C
def seq_str_int(a: int, fmt: str, error: Ptr[bool]) -> str:
pass
@nocapture
@C
def seq_str_uint(a: int, fmt: str, error: Ptr[bool]) -> str:
pass
@nocapture
@C
def seq_str_float(a: float, fmt: str, error: Ptr[bool]) -> str:
pass
@nocapture
@C
def seq_str_str(a: str, fmt: str, error: Ptr[bool]) -> str:
pass
@nocapture
@C
def seq_str_ptr(a: cobj, fmt: str, error: Ptr[bool]) -> str:
pass
@pure
@C
def seq_strdup(a: cobj) -> str:
@ -602,6 +627,16 @@ def free(a: cobj) -> None:
def atoi(a: cobj) -> int:
pass
@nocapture
@C
def strtoll(a: cobj, b: Ptr[cobj], c: i32) -> int:
pass
@nocapture
@C
def strtod(a: cobj, b: Ptr[cobj]) -> float:
pass
# <zlib.h>
@nocapture
@C

View File

@ -0,0 +1,58 @@
# Copyright (C) 2022 Exaloop Inc. <https://exaloop.io>
def _format_error(ret: str):
raise ValueError(f"invalid format specifier: {ret}")
@extend
class int:
def __format__(self, format_spec: str) -> str:
err = False
ret = _C.seq_str_int(self, format_spec, __ptr__(err))
if err:
_format_error(ret)
return ret
@extend
class Int:
def __format__(self, format_spec: str) -> str:
err = False
ret = _C.seq_str_int(self.__int__(), format_spec, __ptr__(err))
if err:
_format_error(ret)
return ret
@extend
class UInt:
def __format__(self, format_spec: str) -> str:
err = False
ret = _C.seq_str_uint(self.__int__(), format_spec, __ptr__(err))
if err:
_format_error(ret)
return ret
@extend
class float:
def __format__(self, format_spec: str) -> str:
err = False
ret = _C.seq_str_float(self, format_spec, __ptr__(err))
if err:
_format_error(ret)
return ret if ret != "-nan" else "nan"
@extend
class str:
def __format__(self, format_spec: str) -> str:
err = False
ret = _C.seq_str_str(self, format_spec, __ptr__(err))
if err:
_format_error(ret)
return ret
@extend
class Ptr:
def __format__(self, format_spec: str) -> str:
err = False
ret = _C.seq_str_ptr(self.as_byte(), format_spec, __ptr__(err))
if err:
_format_error(ret)
return ret

View File

@ -2,24 +2,10 @@
_MAX: Static[int] = 0x7FFFFFFFFFFFFFFF
@pure
@C
def seq_str_str(a: cobj, fmt: cobj, error: cobj) -> str:
pass
@extend
class str:
# Magic methods
def __format__(self, format_spec: str) -> str:
err = byte(0)
ret = seq_str_str(
self.c_str(), format_spec.c_str() if format_spec else cobj(), __ptr__(err)
)
if err:
raise ValueError(f"invalid format specifier: {ret}")
return ret
def __hash__(self) -> int:
h = 0
p, n = self.ptr, self.len
@ -42,7 +28,7 @@ class str:
return self._cmp(other) >= 0
def __repr__(self) -> str:
v = List[str](len(self) + 2)
v = _strbuf(len(self) + 2)
q, qe = "'", "\\'"
found_single = False
@ -80,7 +66,7 @@ class str:
if d:
v.append(d)
v.append(q)
return str.cat(v)
return v.__str__()
def __getitem__(self, idx: int) -> str:
if idx < 0:

View File

@ -4,38 +4,11 @@ from internal.attributes import commutative
from internal.gc import alloc_atomic, free
from internal.types.complex import complex
@pure
@C
def seq_str_float(a: float, fmt: cobj, error: cobj) -> str:
pass
@extend
class float:
def __new__() -> float:
return 0.0
def _from_str(s: str) -> float:
from C import strtod(cobj, Ptr[cobj]) -> float
buf = __array__[byte](32)
n = s.__len__()
need_dyn_alloc = n >= buf.__len__()
p = alloc_atomic(n + 1) if need_dyn_alloc else buf.ptr
str.memcpy(p, s.ptr, n)
p[n] = byte(0)
end = cobj()
result = strtod(p, __ptr__(end))
if need_dyn_alloc:
free(p)
if end != p + n:
raise ValueError(f"could not convert string to float: {s}")
return result
def __new__(what) -> float:
# do not overload! (needed to avoid pyobj conversion)
if isinstance(what, str) or isinstance(what, Optional[str]):
@ -46,15 +19,6 @@ class float:
def __repr__(self) -> str:
return self.__format__("")
def __format__(self, format_spec: str) -> str:
err = byte(0)
ret = seq_str_float(
self, format_spec.c_str() if format_spec else cobj(), __ptr__(err)
)
if err:
raise ValueError(f"invalid format specifier: {ret}")
return ret if ret != "-nan" else "nan"
def __copy__(self) -> float:
return self

View File

@ -3,16 +3,6 @@
from internal.attributes import commutative, associative, distributive
from internal.types.complex import complex
@pure
@C
def seq_str_int(a: int, fmt: cobj, error: cobj) -> str:
pass
@pure
@C
def seq_str_uint(a: int, fmt: cobj, error: cobj) -> str:
pass
@extend
class int:
@pure

View File

@ -449,43 +449,12 @@ class UInt:
def __str__(self) -> str:
return self.__format__("")
def __format__(self, format_spec: str) -> str:
err = byte(0)
ret = seq_str_uint(
self.__int__(), format_spec.c_str() if format_spec else cobj(), __ptr__(err)
)
if err:
raise ValueError(f"invalid format specifier: {ret}")
return ret if ret != "-nan" else "nan"
def popcnt(self) -> int:
return int(Int[N](self)._popcnt())
def len() -> int:
return N
@extend
class Ptr:
def __format__(self, format_spec: str) -> str:
err = byte(0)
ret = seq_str_ptr(
self.as_byte(), format_spec.c_str() if format_spec else cobj(), __ptr__(err)
)
if err:
raise ValueError(f"invalid format specifier: {ret}")
return ret
@extend
class int:
def __format__(self, format_spec: str) -> str:
err = byte(0)
ret = seq_str_int(
self, format_spec.c_str() if format_spec else cobj(), __ptr__(err)
)
if err:
raise ValueError(f"invalid format specifier: {ret}")
return ret
i8 = Int[8]
i16 = Int[16]
i32 = Int[32]

View File

@ -1,10 +1,5 @@
# Copyright (C) 2022 Exaloop Inc. <https://exaloop.io>
@pure
@C
def seq_str_ptr(a: Ptr[byte], fmt: cobj, error: cobj) -> str:
pass
@extend
class Ptr:
@pure

View File

View File

@ -194,6 +194,54 @@ def test_divmod():
assert math.isclose(result[0], exp_result[0])
assert math.isclose(result[1], exp_result[1])
@test
def test_num_from_str():
assert int('0') == 0
assert int('010') == 10
assert int('3\n') == 3
assert int('\r\t\n 42\r\t\n ') == 42
assert int('0101', 2) == 5
assert int('-0101', 2) == -5
assert int('0111', 8) == 73
assert int('-0111', 8) == -73
assert int('-0xabc', 16) == -2748
assert int('0xabc', 16) == 2748
assert int('-0xabc', 16) == -2748
assert int('111', 0) == 111
assert int('-111', 0) == -111
assert int('-0xabc', 0) == -2748
assert int('0xabc', 0) == 2748
assert int('-0xabc', 0) == -2748
try:
int(' 10 a')
assert False
except ValueError as e:
assert str(e) == "invalid literal for int() with base 10: ' 10 a'"
try:
int('')
assert False
except ValueError as e:
assert str(e) == "invalid literal for int() with base 10: ''"
assert float('0') == 0
assert float('3.14') == 3.14
assert float('3\n') == 3
assert float('\r\t\n -4.2\r\t\n ') == -4.2
try:
float(' 3.14 a')
assert False
except ValueError as e:
assert str(e) == "could not convert string to float: ' 3.14 a'"
try:
float('')
assert False
except ValueError as e:
assert str(e) == "could not convert string to float: ''"
@test
def test_files(open_fn):
path = 'build/testfile.txt'
@ -224,6 +272,7 @@ test_map_filter()
test_int_format()
test_reversed()
test_divmod()
test_num_from_str()
test_files(open)
import gzip
test_files(gzip.open)

View File

@ -18,3 +18,8 @@ class Foo:
def ha():
c = C()
print(c.__class__.__name__)
par = 'y'
if len('x') == 1:
par = 'x'

View File

@ -393,9 +393,9 @@ from internal.dlopen import dlext
RT = "./libcodonrt." + dlext()
if RT[-3:] == ".so":
RT = "build/" + RT[2:]
from C import RT.seq_str_int(int, cobj, cobj) -> str as sp
p = cobj(1)
print sp(65, cobj(), p) #: 65
from C import RT.seq_str_int(int, str, Ptr[bool]) -> str as sp
p = False
print sp(65, "", __ptr__(p)) #: 65
#%% import_c_dylib_error,barebones
from C import "".seq_print(str) as sp
@ -438,6 +438,8 @@ print f.__class__.__name__, f.t.__class__.__name__ #: Foo Ptr[B]
a.ha() #: B
print par #: x
#%% import_order,barebones
def foo():
import a