1
0
mirror of https://github.com/exaloop/codon.git synced 2025-06-03 15:03:52 +08:00

Compare commits

..

No commits in common. "develop" and "v0.18.1" have entirely different histories.

21 changed files with 198 additions and 376 deletions

View File

@ -4,39 +4,12 @@ set -e
# setup # setup
cd /github/workspace cd /github/workspace
yum -y update yum -y update
yum -y install gcc gcc-c++ gcc-gfortran make wget openssl-devel bzip2-devel libffi-devel xz-devel zlib-devel yum -y install python3 python3-devel gcc-gfortran
# python
cd /usr/src
wget https://www.openssl.org/source/openssl-1.1.1w.tar.gz
tar xzf openssl-1.1.1w.tar.gz
cd openssl-1.1.1w
./config --prefix=/opt/openssl --openssldir=/opt/openssl no-shared
make -j2
make install
cd /usr/src
wget https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tgz
tar xzf Python-3.11.9.tgz
cd Python-3.11.9
export LD_RUN_PATH=/opt/python311/lib
./configure \
--prefix=/opt/python311 \
--with-openssl=/opt/openssl \
--enable-optimizations \
--enable-shared \
--without-static-libpython \
LDFLAGS="-Wl,-rpath=/opt/python311/lib"
make -j2
make altinstall
export PATH="/opt/python311/bin:$PATH"
alias python=python3.11
# env # env
cd /github/workspace
export PYTHONPATH=$(pwd)/test/python export PYTHONPATH=$(pwd)/test/python
export CODON_PYTHON=$(python test/python/find-python-library.py) export CODON_PYTHON=$(python3 test/python/find-python-library.py)
python -m pip install -Iv pip==21.3.1 numpy==2.0.2 python3 -m pip install -Iv pip==21.3.1 numpy==1.17.5
# deps # deps
if [ ! -d ./llvm ]; then if [ ! -d ./llvm ]; then
@ -57,10 +30,10 @@ cmake --install build --prefix=codon-deploy
# build cython # build cython
export PATH=$PATH:$(pwd)/llvm/bin export PATH=$PATH:$(pwd)/llvm/bin
python -m pip install cython wheel astunparse python3 -m pip install cython wheel astunparse
(cd codon-deploy/python && python setup.py sdist) (cd codon-deploy/python && python3 setup.py sdist)
CODON_DIR=$(pwd)/codon-deploy python -m pip install -v codon-deploy/python/dist/*.gz CODON_DIR=$(pwd)/codon-deploy python3 -m pip install -v codon-deploy/python/dist/*.gz
python test/python/cython_jit.py python3 test/python/cython_jit.py
# test # test
export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH

View File

@ -4,39 +4,12 @@ set -e
# setup # setup
cd /github/workspace cd /github/workspace
yum -y update yum -y update
yum -y install gcc gcc-c++ gcc-gfortran make wget openssl-devel bzip2-devel libffi-devel xz-devel zlib-devel yum -y install python3 python3-devel gcc-gfortran
# python
cd /usr/src
wget https://www.openssl.org/source/openssl-1.1.1w.tar.gz
tar xzf openssl-1.1.1w.tar.gz
cd openssl-1.1.1w
./config --prefix=/opt/openssl --openssldir=/opt/openssl no-shared
make -j2
make install
cd /usr/src
wget https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tgz
tar xzf Python-3.11.9.tgz
cd Python-3.11.9
export LD_RUN_PATH=/opt/python311/lib
./configure \
--prefix=/opt/python311 \
--with-openssl=/opt/openssl \
--enable-optimizations \
--enable-shared \
--without-static-libpython \
LDFLAGS="-Wl,-rpath=/opt/python311/lib"
make -j2
make altinstall
export PATH="/opt/python311/bin:$PATH"
alias python=python3.11
# env # env
cd /github/workspace
export PYTHONPATH=$(pwd)/test/python export PYTHONPATH=$(pwd)/test/python
export CODON_PYTHON=$(python test/python/find-python-library.py) export CODON_PYTHON=$(python3 test/python/find-python-library.py)
python -m pip install -Iv pip==21.3.1 numpy==2.0.2 python3 -m pip install -Iv pip==21.3.1 numpy==1.17.5
# deps # deps
if [ ! -d ./llvm ]; then if [ ! -d ./llvm ]; then
@ -57,10 +30,10 @@ cmake --install build --prefix=codon-deploy
# build cython # build cython
export PATH=$PATH:$(pwd)/llvm/bin export PATH=$PATH:$(pwd)/llvm/bin
python -m pip install cython wheel astunparse python3 -m pip install cython wheel astunparse
(cd codon-deploy/python && python setup.py sdist) (cd codon-deploy/python && python3 setup.py sdist)
CODON_DIR=$(pwd)/codon-deploy python -m pip install -v codon-deploy/python/dist/*.gz CODON_DIR=$(pwd)/codon-deploy python3 -m pip install -v codon-deploy/python/dist/*.gz
python test/python/cython_jit.py python3 test/python/cython_jit.py
# test # test
export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH

View File

@ -121,7 +121,6 @@ jobs:
- name: macOS Setup - name: macOS Setup
if: startsWith(matrix.os, 'macos') if: startsWith(matrix.os, 'macos')
run: | run: |
brew install gcc
echo "LIBEXT=dylib" >> $GITHUB_ENV echo "LIBEXT=dylib" >> $GITHUB_ENV
echo "OS_NAME=osx" >> $GITHUB_ENV echo "OS_NAME=osx" >> $GITHUB_ENV
echo "CODON_SYSTEM_LIBRARIES=$(brew --prefix gcc)/lib/gcc/current" >> $GITHUB_ENV echo "CODON_SYSTEM_LIBRARIES=$(brew --prefix gcc)/lib/gcc/current" >> $GITHUB_ENV
@ -133,7 +132,7 @@ jobs:
run: | run: |
python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade pip setuptools wheel
python -m pip install cython wheel astunparse python -m pip install cython wheel astunparse
python -m pip install --force-reinstall -v "numpy==2.0.2" python -m pip install --force-reinstall -v "numpy==1.26.4"
which python which python
which pip which pip
echo "CODON_PYTHON=$(python test/python/find-python-library.py)" >> $GITHUB_ENV echo "CODON_PYTHON=$(python test/python/find-python-library.py)" >> $GITHUB_ENV
@ -188,7 +187,7 @@ jobs:
- name: Prepare Artifacts - name: Prepare Artifacts
run: | run: |
cp -rf codon-deploy/python/dist . cp -rf codon-deploy/python/dist .
rm -rf codon-deploy/lib/libfmt.a codon-deploy/lib/pkgconfig codon-deploy/lib/cmake codon-deploy/python/codon_jit.egg-info codon-deploy/python/build rm -rf codon-deploy/lib/libfmt.a codon-deploy/lib/pkgconfig codon-deploy/lib/cmake codon-deploy/python/codon.egg-info codon-deploy/python/dist codon-deploy/python/build
tar -czf ${CODON_BUILD_ARCHIVE} codon-deploy tar -czf ${CODON_BUILD_ARCHIVE} codon-deploy
du -sh codon-deploy du -sh codon-deploy

View File

@ -1,10 +1,10 @@
cmake_minimum_required(VERSION 3.14) cmake_minimum_required(VERSION 3.14)
project( project(
Codon Codon
VERSION "0.18.2" VERSION "0.18.1"
HOMEPAGE_URL "https://github.com/exaloop/codon" HOMEPAGE_URL "https://github.com/exaloop/codon"
DESCRIPTION "high-performance, extensible Python compiler") DESCRIPTION "high-performance, extensible Python compiler")
set(CODON_JIT_PYTHON_VERSION "0.3.2") set(CODON_JIT_PYTHON_VERSION "0.3.1")
configure_file("${PROJECT_SOURCE_DIR}/cmake/config.h.in" configure_file("${PROJECT_SOURCE_DIR}/cmake/config.h.in"
"${PROJECT_SOURCE_DIR}/codon/config/config.h") "${PROJECT_SOURCE_DIR}/codon/config/config.h")
configure_file("${PROJECT_SOURCE_DIR}/cmake/config.py.in" configure_file("${PROJECT_SOURCE_DIR}/cmake/config.py.in"
@ -48,8 +48,10 @@ include(${CMAKE_SOURCE_DIR}/cmake/deps.cmake)
set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
if(APPLE) if(APPLE)
set(CMAKE_INSTALL_RPATH "@loader_path;@loader_path/../lib/codon") set(CMAKE_INSTALL_RPATH "@loader_path;@loader_path/../lib/codon")
set(STATIC_LIBCPP "")
else() else()
set(CMAKE_INSTALL_RPATH "$ORIGIN:$ORIGIN/../lib/codon") set(CMAKE_INSTALL_RPATH "$ORIGIN:$ORIGIN/../lib/codon")
set(STATIC_LIBCPP "-static-libstdc++")
endif() endif()
add_executable(peg2cpp codon/util/peg2cpp.cpp) add_executable(peg2cpp codon/util/peg2cpp.cpp)
@ -136,7 +138,7 @@ target_include_directories(codonrt PRIVATE ${backtrace_SOURCE_DIR}
${highway_SOURCE_DIR} ${highway_SOURCE_DIR}
"${gc_SOURCE_DIR}/include" "${gc_SOURCE_DIR}/include"
"${fast_float_SOURCE_DIR}/include" runtime) "${fast_float_SOURCE_DIR}/include" runtime)
target_link_libraries(codonrt PRIVATE fmt omp backtrace LLVMSupport) target_link_libraries(codonrt PRIVATE fmt omp backtrace ${STATIC_LIBCPP} LLVMSupport)
if(APPLE) if(APPLE)
target_link_libraries( target_link_libraries(
codonrt codonrt
@ -432,7 +434,11 @@ llvm_map_components_to_libnames(
TransformUtils TransformUtils
Vectorize Vectorize
Passes) Passes)
target_link_libraries(codonc PRIVATE ${LLVM_LIBS} fmt dl codonrt) if(APPLE)
target_link_libraries(codonc PRIVATE ${LLVM_LIBS} fmt dl codonrt)
else()
target_link_libraries(codonc PRIVATE ${STATIC_LIBCPP} ${LLVM_LIBS} fmt dl codonrt)
endif()
# Gather headers # Gather headers
add_custom_target( add_custom_target(
@ -476,13 +482,13 @@ add_dependencies(libs codonrt codonc)
# Codon command-line tool # Codon command-line tool
add_executable(codon codon/app/main.cpp) add_executable(codon codon/app/main.cpp)
target_link_libraries(codon PUBLIC fmt codonc codon_jupyter Threads::Threads) target_link_libraries(codon PUBLIC ${STATIC_LIBCPP} fmt codonc codon_jupyter Threads::Threads)
# Codon test Download and unpack googletest at configure time # Codon test Download and unpack googletest at configure time
include(FetchContent) include(FetchContent)
FetchContent_Declare( FetchContent_Declare(
googletest googletest
URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
) )
# For Windows: Prevent overriding the parent project's compiler/linker settings # For Windows: Prevent overriding the parent project's compiler/linker settings
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)

View File

@ -149,7 +149,7 @@ print(total)
``` ```
Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic
reduction to avoid race conditions. Learn more in the [multithreading docs](https://docs.exaloop.io/codon/advanced/parallel). reduction to avoid race conditions. Learn more in the [multitheading docs](advanced/parallel.md).
Codon also supports writing and executing GPU kernels. Here's an example that computes the Codon also supports writing and executing GPU kernels. Here's an example that computes the
[Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set): [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set):

View File

@ -1,8 +1,8 @@
set(CPM_DOWNLOAD_VERSION 0.40.8) set(CPM_DOWNLOAD_VERSION 0.32.3)
set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake") set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION})) if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION}))
message(STATUS "Downloading CPM.cmake...") message(STATUS "Downloading CPM.cmake...")
file(DOWNLOAD https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake ${CPM_DOWNLOAD_LOCATION}) file(DOWNLOAD https://github.com/TheLartians/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake ${CPM_DOWNLOAD_LOCATION})
endif() endif()
include(${CPM_DOWNLOAD_LOCATION}) include(${CPM_DOWNLOAD_LOCATION})
@ -77,9 +77,9 @@ endif()
CPMAddPackage( CPMAddPackage(
NAME bdwgc NAME bdwgc
GITHUB_REPOSITORY "exaloop/bdwgc" GITHUB_REPOSITORY "ivmai/bdwgc"
VERSION 8.0.5 VERSION 8.0.5
GIT_TAG e16c67244aff26802203060422545d38305e0160 GIT_TAG d0ba209660ea8c663e06d9a68332ba5f42da54ba
EXCLUDE_FROM_ALL YES EXCLUDE_FROM_ALL YES
OPTIONS "CMAKE_POSITION_INDEPENDENT_CODE ON" OPTIONS "CMAKE_POSITION_INDEPENDENT_CODE ON"
"BUILD_SHARED_LIBS OFF" "BUILD_SHARED_LIBS OFF"
@ -169,7 +169,7 @@ if(NOT APPLE)
CPMAddPackage( CPMAddPackage(
NAME openblas NAME openblas
GITHUB_REPOSITORY "OpenMathLib/OpenBLAS" GITHUB_REPOSITORY "OpenMathLib/OpenBLAS"
GIT_TAG v0.3.29 GIT_TAG v0.3.28
EXCLUDE_FROM_ALL YES EXCLUDE_FROM_ALL YES
OPTIONS "DYNAMIC_ARCH ON" OPTIONS "DYNAMIC_ARCH ON"
"BUILD_TESTING OFF" "BUILD_TESTING OFF"

View File

@ -11,7 +11,6 @@
#include <unordered_map> #include <unordered_map>
#include <vector> #include <vector>
#include "codon/cir/util/format.h"
#include "codon/compiler/compiler.h" #include "codon/compiler/compiler.h"
#include "codon/compiler/error.h" #include "codon/compiler/error.h"
#include "codon/compiler/jit.h" #include "codon/compiler/jit.h"
@ -88,7 +87,7 @@ void initLogFlags(const llvm::cl::opt<std::string> &log) {
codon::getLogger().parse(std::string(d)); codon::getLogger().parse(std::string(d));
} }
enum BuildKind { LLVM, Bitcode, Object, Executable, Library, PyExtension, Detect, CIR }; enum BuildKind { LLVM, Bitcode, Object, Executable, Library, PyExtension, Detect };
enum OptMode { Debug, Release }; enum OptMode { Debug, Release };
enum Numerics { C, Python }; enum Numerics { C, Python };
} // namespace } // namespace
@ -334,7 +333,6 @@ int buildMode(const std::vector<const char *> &args, const std::string &argv0) {
clEnumValN(Executable, "exe", "Generate executable"), clEnumValN(Executable, "exe", "Generate executable"),
clEnumValN(Library, "lib", "Generate shared library"), clEnumValN(Library, "lib", "Generate shared library"),
clEnumValN(PyExtension, "pyext", "Generate Python extension module"), clEnumValN(PyExtension, "pyext", "Generate Python extension module"),
clEnumValN(CIR, "cir", "Generate Codon Intermediate Representation"),
clEnumValN(Detect, "detect", clEnumValN(Detect, "detect",
"Detect output type based on output file extension")), "Detect output type based on output file extension")),
llvm::cl::init(Detect)); llvm::cl::init(Detect));
@ -374,9 +372,6 @@ int buildMode(const std::vector<const char *> &args, const std::string &argv0) {
case BuildKind::Detect: case BuildKind::Detect:
extension = ""; extension = "";
break; break;
case BuildKind::CIR:
extension = ".cir";
break;
default: default:
seqassertn(0, "unknown build kind"); seqassertn(0, "unknown build kind");
} }
@ -406,11 +401,6 @@ int buildMode(const std::vector<const char *> &args, const std::string &argv0) {
compiler->getLLVMVisitor()->writeToPythonExtension(*compiler->getCache()->pyModule, compiler->getLLVMVisitor()->writeToPythonExtension(*compiler->getCache()->pyModule,
filename); filename);
break; break;
case BuildKind::CIR: {
std::ofstream out(filename);
codon::ir::util::format(out, compiler->getModule());
break;
}
case BuildKind::Detect: case BuildKind::Detect:
compiler->getLLVMVisitor()->compile(filename, argv0, libsVec, lflags); compiler->getLLVMVisitor()->compile(filename, argv0, libsVec, lflags);
break; break;

View File

@ -376,9 +376,9 @@ void LLVMVisitor::writeToObjectFile(const std::string &filename, bool pic) {
llvm::TargetLibraryInfoImpl tlii(llvm::Triple(M->getTargetTriple())); llvm::TargetLibraryInfoImpl tlii(llvm::Triple(M->getTargetTriple()));
pm.add(new llvm::TargetLibraryInfoWrapperPass(tlii)); pm.add(new llvm::TargetLibraryInfoWrapperPass(tlii));
if (machine->addPassesToEmitFile(pm, *os, nullptr, llvm::CGFT_ObjectFile, seqassertn(!machine->addPassesToEmitFile(pm, *os, nullptr, llvm::CGFT_ObjectFile,
/*DisableVerify=*/true, mmiwp)) /*DisableVerify=*/true, mmiwp),
seqassertn(false, "could not add passes"); "could not add passes");
const_cast<llvm::TargetLoweringObjectFile *>(llvmtm.getObjFileLowering()) const_cast<llvm::TargetLoweringObjectFile *>(llvmtm.getObjFileLowering())
->Initialize(mmiwp->getMMI().getContext(), *machine); ->Initialize(mmiwp->getMMI().getContext(), *machine);
pm.run(*M); pm.run(*M);

View File

@ -263,21 +263,21 @@ ir::types::Type *JIT::PythonData::getCObjType(ir::Module *M) {
return cobj; return cobj;
} }
JIT::JITResult JIT::executeSafe(const std::string &code, const std::string &file, JITResult JIT::executeSafe(const std::string &code, const std::string &file, int line,
int line, bool debug) { bool debug) {
auto result = execute(code, file, line, debug); auto result = execute(code, file, line, debug);
if (auto err = result.takeError()) { if (auto err = result.takeError()) {
auto errorInfo = llvm::toString(std::move(err)); auto errorInfo = llvm::toString(std::move(err));
return JITResult::error(errorInfo); return JITResult::error(errorInfo);
} }
return JITResult::success(); return JITResult::success(nullptr);
} }
JIT::JITResult JIT::executePython(const std::string &name, JITResult JIT::executePython(const std::string &name,
const std::vector<std::string> &types, const std::vector<std::string> &types,
const std::string &pyModule, const std::string &pyModule,
const std::vector<std::string> &pyVars, void *arg, const std::vector<std::string> &pyVars, void *arg,
bool debug) { bool debug) {
auto key = buildKey(name, types); auto key = buildKey(name, types);
auto &cache = pydata->cache; auto &cache = pydata->cache;
auto it = cache.find(key); auto it = cache.find(key);
@ -322,48 +322,26 @@ JIT::JITResult JIT::executePython(const std::string &name,
} }
} }
} // namespace jit JIT *jitInit(const std::string &name) {
} // namespace codon auto jit = new JIT(name);
void *jit_init(char *name) {
auto jit = new codon::jit::JIT(std::string(name));
llvm::cantFail(jit->init()); llvm::cantFail(jit->init());
return jit; return jit;
} }
void jit_exit(void *jit) { delete ((codon::jit::JIT *)jit); } JITResult jitExecutePython(JIT *jit, const std::string &name,
const std::vector<std::string> &types,
CJITResult jit_execute_python(void *jit, char *name, char **types, size_t types_size, const std::string &pyModule,
char *pyModule, char **py_vars, size_t py_vars_size, const std::vector<std::string> &pyVars, void *arg,
void *arg, uint8_t debug) { bool debug) {
std::vector<std::string> cppTypes; return jit->executePython(name, types, pyModule, pyVars, arg, debug);
cppTypes.reserve(types_size);
for (size_t i = 0; i < types_size; i++)
cppTypes.emplace_back(types[i]);
std::vector<std::string> cppPyVars;
cppPyVars.reserve(py_vars_size);
for (size_t i = 0; i < py_vars_size; i++)
cppPyVars.emplace_back(py_vars[i]);
auto t = ((codon::jit::JIT *)jit)
->executePython(std::string(name), cppTypes, std::string(pyModule),
cppPyVars, arg, bool(debug));
void *result = t.result;
char *message =
t.message.empty() ? nullptr : strndup(t.message.c_str(), t.message.size());
return {result, message};
} }
CJITResult jit_execute_safe(void *jit, char *code, char *file, int32_t line, JITResult jitExecuteSafe(JIT *jit, const std::string &code, const std::string &file,
uint8_t debug) { int line, bool debug) {
auto t = ((codon::jit::JIT *)jit) return jit->executeSafe(code, file, line, debug);
->executeSafe(std::string(code), std::string(file), line, bool(debug));
void *result = t.result;
char *message =
t.message.empty() ? nullptr : strndup(t.message.c_str(), t.message.size());
return {result, message};
} }
char *get_jit_library() { std::string getJITLibrary() { return ast::library_path(); }
auto t = codon::ast::library_path();
return strndup(t.c_str(), t.size()); } // namespace jit
} } // namespace codon

View File

@ -31,15 +31,6 @@ public:
ir::types::Type *getCObjType(ir::Module *M); ir::types::Type *getCObjType(ir::Module *M);
}; };
struct JITResult {
void *result;
std::string message;
operator bool() const { return message.empty(); }
static JITResult success(void *result = nullptr) { return {result, ""}; }
static JITResult error(const std::string &message) { return {nullptr, message}; }
};
private: private:
std::unique_ptr<Compiler> compiler; std::unique_ptr<Compiler> compiler;
std::unique_ptr<Engine> engine; std::unique_ptr<Engine> engine;

View File

@ -2,30 +2,35 @@
#pragma once #pragma once
#include <stddef.h> #include <string>
#include <stdint.h> #include <vector>
#ifdef __cplusplus namespace codon {
extern "C" { namespace jit {
#endif
struct CJITResult { class JIT;
struct JITResult {
void *result; void *result;
char *error; std::string message;
operator bool() const { return message.empty(); }
static JITResult success(void *result) { return {result, ""}; }
static JITResult error(const std::string &message) { return {nullptr, message}; }
}; };
void *jit_init(char *name); JIT *jitInit(const std::string &name);
void jit_exit(void *jit);
struct CJITResult jit_execute_python(void *jit, char *name, char **types, JITResult jitExecutePython(JIT *jit, const std::string &name,
size_t types_size, char *pyModule, char **py_vars, const std::vector<std::string> &types,
size_t py_vars_size, void *arg, uint8_t debug); const std::string &pyModule,
const std::vector<std::string> &pyVars, void *arg,
bool debug);
struct CJITResult jit_execute_safe(void *jit, char *code, char *file, int32_t line, JITResult jitExecuteSafe(JIT *jit, const std::string &code, const std::string &file,
uint8_t debug); int line, bool debug);
char *get_jit_library(); std::string getJITLibrary();
#ifdef __cplusplus } // namespace jit
} } // namespace codon
#endif

View File

@ -124,7 +124,7 @@ print(total)
``` ```
Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic
reduction to avoid race conditions. Learn more in the [multithreading docs](advanced/parallel.md). reduction to avoid race conditions. Learn more in the [multitheading docs](advanced/parallel.md).
Codon also supports writing and executing GPU kernels. Here's an example that computes the Codon also supports writing and executing GPU kernels. Here's an example that computes the
[Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set): [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set):

View File

@ -1,7 +1,5 @@
# Copyright (C) 2022-2025 Exaloop Inc. <https://exaloop.io> # Copyright (C) 2022-2025 Exaloop Inc. <https://exaloop.io>
__all__ = [ __all__ = ["jit", "convert", "JITError"]
"jit", "convert", "JITError", "JITWrapper", "_jit_register_fn", "_jit"
]
from .decorator import jit, convert, execute, JITError, JITWrapper, _jit_register_fn, _jit_callback_fn, _jit from .decorator import jit, convert, execute, JITError

View File

@ -23,14 +23,16 @@ if "CODON_PATH" not in os.environ:
if codon_lib_path: if codon_lib_path:
codon_path.append(Path(codon_lib_path).parent / "stdlib") codon_path.append(Path(codon_lib_path).parent / "stdlib")
codon_path.append( codon_path.append(
Path(os.path.expanduser("~")) / ".codon" / "lib" / "codon" / "stdlib") Path(os.path.expanduser("~")) / ".codon" / "lib" / "codon" / "stdlib"
)
for path in codon_path: for path in codon_path:
if path.exists(): if path.exists():
os.environ["CODON_PATH"] = str(path.resolve()) os.environ["CODON_PATH"] = str(path.resolve())
break break
else: else:
raise RuntimeError( raise RuntimeError(
"Cannot locate Codon. Please install Codon or set CODON_PATH.") "Cannot locate Codon. Please install Codon or set CODON_PATH."
)
pod_conversions = { pod_conversions = {
type(None): "pyobj", type(None): "pyobj",
@ -59,6 +61,7 @@ pod_conversions = {
custom_conversions = {} custom_conversions = {}
_error_msgs = set() _error_msgs = set()
def _common_type(t, debug, sample_size): def _common_type(t, debug, sample_size):
sub, is_optional = None, False sub, is_optional = None, False
for i in itertools.islice(t, sample_size): for i in itertools.islice(t, sample_size):
@ -73,6 +76,7 @@ def _common_type(t, debug, sample_size):
sub = "Optional[{}]".format(sub) sub = "Optional[{}]".format(sub)
return sub if sub else "pyobj" return sub if sub else "pyobj"
def _codon_type(arg, **kwargs): def _codon_type(arg, **kwargs):
t = type(arg) t = type(arg)
@ -84,11 +88,11 @@ def _codon_type(arg, **kwargs):
if issubclass(t, set): if issubclass(t, set):
return "Set[{}]".format(_common_type(arg, **kwargs)) return "Set[{}]".format(_common_type(arg, **kwargs))
if issubclass(t, dict): if issubclass(t, dict):
return "Dict[{},{}]".format(_common_type(arg.keys(), **kwargs), return "Dict[{},{}]".format(
_common_type(arg.values(), **kwargs)) _common_type(arg.keys(), **kwargs), _common_type(arg.values(), **kwargs)
)
if issubclass(t, tuple): if issubclass(t, tuple):
return "Tuple[{}]".format(",".join( return "Tuple[{}]".format(",".join(_codon_type(a, **kwargs) for a in arg))
_codon_type(a, **kwargs) for a in arg))
if issubclass(t, np.ndarray): if issubclass(t, np.ndarray):
if arg.dtype == np.bool_: if arg.dtype == np.bool_:
dtype = "bool" dtype = "bool"
@ -130,8 +134,7 @@ def _codon_type(arg, **kwargs):
s = custom_conversions.get(t, "") s = custom_conversions.get(t, "")
if s: if s:
j = ",".join( j = ",".join(_codon_type(getattr(arg, slot), **kwargs) for slot in t.__slots__)
_codon_type(getattr(arg, slot), **kwargs) for slot in t.__slots__)
return "{}[{}]".format(s, j) return "{}[{}]".format(s, j)
debug = kwargs.get("debug", None) debug = kwargs.get("debug", None)
@ -142,22 +145,28 @@ def _codon_type(arg, **kwargs):
_error_msgs.add(msg) _error_msgs.add(msg)
return "pyobj" return "pyobj"
def _codon_types(args, **kwargs): def _codon_types(args, **kwargs):
return tuple(_codon_type(arg, **kwargs) for arg in args) return tuple(_codon_type(arg, **kwargs) for arg in args)
def _reset_jit(): def _reset_jit():
global _jit global _jit
_jit = JITWrapper() _jit = JITWrapper()
init_code = ("from internal.python import " init_code = (
"setup_decorator, PyTuple_GetItem, PyObject_GetAttrString\n" "from internal.python import "
"setup_decorator()\n" "setup_decorator, PyTuple_GetItem, PyObject_GetAttrString\n"
"import numpy as np\n" "setup_decorator()\n"
"import numpy.pybridge\n") "import numpy as np\n"
"import numpy.pybridge\n"
)
_jit.execute(init_code, "", 0, False) _jit.execute(init_code, "", 0, False)
return _jit return _jit
_jit = _reset_jit() _jit = _reset_jit()
class RewriteFunctionArgs(ast.NodeTransformer): class RewriteFunctionArgs(ast.NodeTransformer):
def __init__(self, args): def __init__(self, args):
self.args = args self.args = args
@ -167,6 +176,7 @@ class RewriteFunctionArgs(ast.NodeTransformer):
node.args.args.append(ast.arg(arg=a, annotation=None)) node.args.args.append(ast.arg(arg=a, annotation=None))
return node return node
def _obj_to_str(obj, **kwargs) -> str: def _obj_to_str(obj, **kwargs) -> str:
if inspect.isclass(obj): if inspect.isclass(obj):
lines = inspect.getsourcelines(obj)[0] lines = inspect.getsourcelines(obj)[0]
@ -175,10 +185,8 @@ def _obj_to_str(obj, **kwargs) -> str:
obj_name = obj.__name__ obj_name = obj.__name__
elif callable(obj) or isinstance(obj, str): elif callable(obj) or isinstance(obj, str):
is_str = isinstance(obj, str) is_str = isinstance(obj, str)
lines = [i + '\n' for i in obj.split('\n') lines = [i + '\n' for i in obj.split('\n')] if is_str else inspect.getsourcelines(obj)[0]
] if is_str else inspect.getsourcelines(obj)[0] if not is_str: lines = lines[1:]
if not is_str:
lines = lines[1:]
obj_str = textwrap.dedent(''.join(lines)) obj_str = textwrap.dedent(''.join(lines))
pyvars = kwargs.get("pyvars", None) pyvars = kwargs.get("pyvars", None)
@ -187,7 +195,8 @@ def _obj_to_str(obj, **kwargs) -> str:
if not isinstance(i, str): if not isinstance(i, str):
raise ValueError("pyvars only takes string literals") raise ValueError("pyvars only takes string literals")
node = ast.fix_missing_locations( node = ast.fix_missing_locations(
RewriteFunctionArgs(pyvars).visit(ast.parse(obj_str))) RewriteFunctionArgs(pyvars).visit(ast.parse(obj_str))
)
obj_str = astunparse.unparse(node) obj_str = astunparse.unparse(node)
if is_str: if is_str:
try: try:
@ -197,23 +206,28 @@ def _obj_to_str(obj, **kwargs) -> str:
else: else:
obj_name = obj.__name__ obj_name = obj.__name__
else: else:
raise TypeError("Function or class expected, got " + raise TypeError("Function or class expected, got " + type(obj).__name__)
type(obj).__name__)
return obj_name, obj_str.replace("_@par", "@par") return obj_name, obj_str.replace("_@par", "@par")
def _parse_decorated(obj, **kwargs): def _parse_decorated(obj, **kwargs):
return _obj_to_str(obj, **kwargs) return _obj_to_str(obj, **kwargs)
def convert(t): def convert(t):
if not hasattr(t, "__slots__"): if not hasattr(t, "__slots__"):
raise JITError("class '{}' does not have '__slots__' attribute".format( raise JITError("class '{}' does not have '__slots__' attribute".format(str(t)))
str(t)))
name = t.__name__ name = t.__name__
slots = t.__slots__ slots = t.__slots__
code = ("@tuple\n" code = (
"class " + name + "[" + "@tuple\n"
",".join("T{}".format(i) for i in range(len(slots))) + "]:\n") "class "
+ name
+ "["
+ ",".join("T{}".format(i) for i in range(len(slots)))
+ "]:\n"
)
for i, slot in enumerate(slots): for i, slot in enumerate(slots):
code += " {}: T{}\n".format(slot, i) code += " {}: T{}\n".format(slot, i)
@ -221,14 +235,17 @@ def convert(t):
code += " def __from_py__(p: cobj):\n" code += " def __from_py__(p: cobj):\n"
for i, slot in enumerate(slots): for i, slot in enumerate(slots):
code += " a{} = T{}.__from_py__(PyObject_GetAttrString(p, '{}'.ptr))\n".format( code += " a{} = T{}.__from_py__(PyObject_GetAttrString(p, '{}'.ptr))\n".format(
i, i, slot) i, i, slot
)
code += " return {}({})\n".format( code += " return {}({})\n".format(
name, ", ".join("a{}".format(i) for i in range(len(slots)))) name, ", ".join("a{}".format(i) for i in range(len(slots)))
)
_jit.execute(code, "", 0, False) _jit.execute(code, "", 0, False)
custom_conversions[t] = name custom_conversions[t] = name
return t return t
def _jit_register_fn(f, pyvars, debug): def _jit_register_fn(f, pyvars, debug):
try: try:
obj_name, obj_str = _parse_decorated(f, pyvars=pyvars) obj_name, obj_str = _parse_decorated(f, pyvars=pyvars)
@ -241,46 +258,29 @@ def _jit_register_fn(f, pyvars, debug):
_reset_jit() _reset_jit()
raise raise
def _jit_callback_fn(fn, def _jit_callback_fn(obj_name, module, debug=None, sample_size=5, pyvars=None, *args, **kwargs):
obj_name,
module,
debug=None,
sample_size=5,
pyvars=None,
*args,
**kwargs):
if fn is not None:
sig = inspect.signature(fn)
bound_args = sig.bind(*args, **kwargs)
bound_args.apply_defaults()
args = tuple(bound_args.arguments[param] for param in sig.parameters)
else:
args = (*args, *kwargs.values())
try: try:
args = (*args, *kwargs.values())
types = _codon_types(args, debug=debug, sample_size=sample_size) types = _codon_types(args, debug=debug, sample_size=sample_size)
if debug: if debug:
print("[python] {}({})".format(obj_name, list(types)), print("[python] {}({})".format(obj_name, list(types)), file=sys.stderr)
file=sys.stderr) return _jit.run_wrapper(
return _jit.run_wrapper(obj_name, list(types), module, list(pyvars), obj_name, list(types), module, list(pyvars), args, 1 if debug else 0
args, 1 if debug else 0) )
except JITError: except JITError:
_reset_jit() _reset_jit()
raise raise
def _jit_str_fn(fstr, debug=None, sample_size=5, pyvars=None): def _jit_str_fn(fstr, debug=None, sample_size=5, pyvars=None):
obj_name = _jit_register_fn(fstr, pyvars, debug) obj_name = _jit_register_fn(fstr, pyvars, debug)
def wrapped(*args, **kwargs): def wrapped(*args, **kwargs):
return _jit_callback_fn(None, obj_name, "__main__", debug, sample_size, return _jit_callback_fn(obj_name, "__main__", debug, sample_size, pyvars, *args, **kwargs)
pyvars, *args, **kwargs)
return wrapped return wrapped
def jit(fn=None, debug=None, sample_size=5, pyvars=None): def jit(fn=None, debug=None, sample_size=5, pyvars=None):
if not pyvars: if not pyvars:
pyvars = [] pyvars = []
if not isinstance(pyvars, list): if not isinstance(pyvars, list):
raise ArgumentError("pyvars must be a list") raise ArgumentError("pyvars must be a list")
@ -289,16 +289,13 @@ def jit(fn=None, debug=None, sample_size=5, pyvars=None):
def _decorate(f): def _decorate(f):
obj_name = _jit_register_fn(f, pyvars, debug) obj_name = _jit_register_fn(f, pyvars, debug)
@functools.wraps(f) @functools.wraps(f)
def wrapped(*args, **kwargs): def wrapped(*args, **kwargs):
return _jit_callback_fn(f, obj_name, f.__module__, debug, return _jit_callback_fn(obj_name, f.__module__, debug, sample_size, pyvars, *args, **kwargs)
sample_size, pyvars, *args, **kwargs)
return wrapped return wrapped
return _decorate(fn) if fn else _decorate return _decorate(fn) if fn else _decorate
def execute(code, debug=False): def execute(code, debug=False):
try: try:
_jit.execute(code, "<internal>", 0, int(debug)) _jit.execute(code, "<internal>", 0, int(debug))

View File

@ -1,22 +1,16 @@
# Copyright (C) 2022-2025 Exaloop Inc. <https://exaloop.io> # Copyright (C) 2022-2025 Exaloop Inc. <https://exaloop.io>
from libc.stdint cimport int32_t, uint8_t from libcpp.string cimport string
from libcpp.vector cimport vector
cdef extern from "codon/compiler/jit_extern.h": cdef extern from "codon/compiler/jit_extern.h" namespace "codon::jit":
cdef struct CJITResult: cdef cppclass JIT
cdef cppclass JITResult:
void *result void *result
char *error string message
bint operator bool()
void *jit_init(char *name) JIT *jitInit(string)
void jit_exit(void *jit) JITResult jitExecuteSafe(JIT*, string, string, int, char)
JITResult jitExecutePython(JIT*, string, vector[string], string, vector[string], object, char)
cdef char *get_jit_library() string getJITLibrary()
cdef CJITResult jit_execute_safe(
void *jit, char *code, char *file, int32_t line, uint8_t debug
)
cdef CJITResult jit_execute_python(
void *jit, char *name, char **types, size_t types_size,
char *pyModule, char **py_vars, size_t py_vars_size,
void *arg, uint8_t debug
)

View File

@ -1,82 +1,45 @@
# Copyright (C) 2022-2025 Exaloop Inc. <https://exaloop.io> # Copyright (C) 2022-2025 Exaloop Inc. <https://exaloop.io>
# distutils: language=c # distutils: language=c++
# cython: language_level=3 # cython: language_level=3
# cython: c_string_type=unicode # cython: c_string_type=unicode
# cython: c_string_encoding=utf8 # cython: c_string_encoding=utf8
from libcpp.string cimport string
from libcpp.vector cimport vector
cimport codon.jit cimport codon.jit
from libc.stdlib cimport malloc, calloc, free
from libc.string cimport strcpy
from libc.stdint cimport int32_t, uint8_t
class JITError(Exception): class JITError(Exception):
pass pass
cdef str get_free_str(char *s):
cdef bytes py_s
try:
py_s = s
return py_s.decode('utf-8')
finally:
free(s)
cdef class JITWrapper: cdef class JITWrapper:
cdef void* jit cdef codon.jit.JIT* jit
def __cinit__(self): def __cinit__(self):
self.jit = codon.jit.jit_init(b"codon jit") self.jit = codon.jit.jitInit(b"codon jit")
def __dealloc__(self): def __dealloc__(self):
codon.jit.jit_exit(self.jit) del self.jit
def execute(self, code: str, filename: str, fileno: int, debug) -> str: def execute(self, code: str, filename: str, fileno: int, debug: char) -> str:
result = codon.jit.jit_execute_safe( result = codon.jit.jitExecuteSafe(self.jit, code, filename, fileno, <char>debug)
self.jit, code.encode('utf-8'), filename.encode('utf-8'), fileno, <uint8_t>debug if <bint>result:
)
if result.error is NULL:
return None return None
else: else:
msg = get_free_str(result.error) raise JITError(result.message)
raise JITError(msg)
def run_wrapper(self, name: str, types: list[str], module: str, pyvars: list[str], args, debug) -> object:
cdef char** c_types = <char**>calloc(len(types), sizeof(char*))
cdef char** c_pyvars = <char**>calloc(len(pyvars), sizeof(char*))
if not c_types or not c_pyvars:
raise JITError("Cython allocation failed")
try:
for i, s in enumerate(types):
bytes = s.encode('utf-8')
c_types[i] = <char*>malloc(len(bytes) + 1)
strcpy(c_types[i], bytes)
for i, s in enumerate(pyvars):
bytes = s.encode('utf-8')
c_pyvars[i] = <char*>malloc(len(bytes) + 1)
strcpy(c_pyvars[i], bytes)
result = codon.jit.jit_execute_python(
self.jit, name.encode('utf-8'), c_types, len(types),
module.encode('utf-8'), c_pyvars, len(pyvars),
<void *>args, <uint8_t>debug
)
if result.error is NULL:
return <object>result.result
else:
msg = get_free_str(result.error)
raise JITError(msg)
finally:
for i in range(len(types)):
free(c_types[i])
free(c_types)
for i in range(len(pyvars)):
free(c_pyvars[i])
free(c_pyvars)
def run_wrapper(self, name: str, types: list[str], module: str, pyvars: list[str], args, debug: char) -> object:
cdef vector[string] types_vec = types
cdef vector[string] pyvars_vec = pyvars
result = codon.jit.jitExecutePython(
self.jit, name, types_vec, module, pyvars_vec, <object>args, <char>debug
)
if <bint>result:
return <object>result.result
else:
raise JITError(result.message)
def codon_library(): def codon_library():
cdef char* c = codon.jit.get_jit_library() return codon.jit.getJITLibrary()
return get_free_str(c)

View File

@ -67,7 +67,7 @@ jit_extension = Extension(
"codon.codon_jit", "codon.codon_jit",
sources=["codon/jit.pyx"], sources=["codon/jit.pyx"],
libraries=libraries, libraries=libraries,
language="c", language="c++",
extra_compile_args=["-w"], extra_compile_args=["-w"],
extra_link_args=linker_args, extra_link_args=linker_args,
include_dirs=[str(codon_path / "include")], include_dirs=[str(codon_path / "include")],

View File

@ -24,7 +24,6 @@ PyFloat_AsDouble = Function[[cobj], float](cobj())
PyFloat_FromDouble = Function[[float], cobj](cobj()) PyFloat_FromDouble = Function[[float], cobj](cobj())
PyBool_FromLong = Function[[int], cobj](cobj()) PyBool_FromLong = Function[[int], cobj](cobj())
PyBytes_AsString = Function[[cobj], cobj](cobj()) PyBytes_AsString = Function[[cobj], cobj](cobj())
PyBytes_Size = Function[[cobj], int](cobj())
PyList_New = Function[[int], cobj](cobj()) PyList_New = Function[[int], cobj](cobj())
PyList_Size = Function[[cobj], int](cobj()) PyList_Size = Function[[cobj], int](cobj())
PyList_GetItem = Function[[cobj, int], cobj](cobj()) PyList_GetItem = Function[[cobj, int], cobj](cobj())
@ -131,7 +130,6 @@ PyLong_Type = cobj()
PyFloat_Type = cobj() PyFloat_Type = cobj()
PyBool_Type = cobj() PyBool_Type = cobj()
PyUnicode_Type = cobj() PyUnicode_Type = cobj()
PyBytes_Type = cobj()
PyComplex_Type = cobj() PyComplex_Type = cobj()
PyList_Type = cobj() PyList_Type = cobj()
PyDict_Type = cobj() PyDict_Type = cobj()
@ -215,7 +213,6 @@ def init_handles_dlopen(py_handle: cobj):
global PyFloat_FromDouble global PyFloat_FromDouble
global PyBool_FromLong global PyBool_FromLong
global PyBytes_AsString global PyBytes_AsString
global PyBytes_Size
global PyList_New global PyList_New
global PyList_Size global PyList_Size
global PyList_GetItem global PyList_GetItem
@ -306,7 +303,6 @@ def init_handles_dlopen(py_handle: cobj):
global PyFloat_Type global PyFloat_Type
global PyBool_Type global PyBool_Type
global PyUnicode_Type global PyUnicode_Type
global PyBytes_Type
global PyComplex_Type global PyComplex_Type
global PyList_Type global PyList_Type
global PyDict_Type global PyDict_Type
@ -351,7 +347,6 @@ def init_handles_dlopen(py_handle: cobj):
PyFloat_FromDouble = dlsym(py_handle, "PyFloat_FromDouble") PyFloat_FromDouble = dlsym(py_handle, "PyFloat_FromDouble")
PyBool_FromLong = dlsym(py_handle, "PyBool_FromLong") PyBool_FromLong = dlsym(py_handle, "PyBool_FromLong")
PyBytes_AsString = dlsym(py_handle, "PyBytes_AsString") PyBytes_AsString = dlsym(py_handle, "PyBytes_AsString")
PyBytes_Size = dlsym(py_handle, "PyBytes_Size")
PyList_New = dlsym(py_handle, "PyList_New") PyList_New = dlsym(py_handle, "PyList_New")
PyList_Size = dlsym(py_handle, "PyList_Size") PyList_Size = dlsym(py_handle, "PyList_Size")
PyList_GetItem = dlsym(py_handle, "PyList_GetItem") PyList_GetItem = dlsym(py_handle, "PyList_GetItem")
@ -442,7 +437,6 @@ def init_handles_dlopen(py_handle: cobj):
PyFloat_Type = dlsym(py_handle, "PyFloat_Type") PyFloat_Type = dlsym(py_handle, "PyFloat_Type")
PyBool_Type = dlsym(py_handle, "PyBool_Type") PyBool_Type = dlsym(py_handle, "PyBool_Type")
PyUnicode_Type = dlsym(py_handle, "PyUnicode_Type") PyUnicode_Type = dlsym(py_handle, "PyUnicode_Type")
PyBytes_Type = dlsym(py_handle, "PyBytes_Type")
PyComplex_Type = dlsym(py_handle, "PyComplex_Type") PyComplex_Type = dlsym(py_handle, "PyComplex_Type")
PyList_Type = dlsym(py_handle, "PyList_Type") PyList_Type = dlsym(py_handle, "PyList_Type")
PyDict_Type = dlsym(py_handle, "PyDict_Type") PyDict_Type = dlsym(py_handle, "PyDict_Type")
@ -488,7 +482,6 @@ def init_handles_static():
from C import PyFloat_FromDouble(float) -> cobj as _PyFloat_FromDouble from C import PyFloat_FromDouble(float) -> cobj as _PyFloat_FromDouble
from C import PyBool_FromLong(int) -> cobj as _PyBool_FromLong from C import PyBool_FromLong(int) -> cobj as _PyBool_FromLong
from C import PyBytes_AsString(cobj) -> cobj as _PyBytes_AsString from C import PyBytes_AsString(cobj) -> cobj as _PyBytes_AsString
from C import PyBytes_Size(cobj) -> int as _PyBytes_Size
from C import PyList_New(int) -> cobj as _PyList_New from C import PyList_New(int) -> cobj as _PyList_New
from C import PyList_Size(cobj) -> int as _PyList_Size from C import PyList_Size(cobj) -> int as _PyList_Size
from C import PyList_GetItem(cobj, int) -> cobj as _PyList_GetItem from C import PyList_GetItem(cobj, int) -> cobj as _PyList_GetItem
@ -579,7 +572,6 @@ def init_handles_static():
from C import PyFloat_Type: cobj as _PyFloat_Type from C import PyFloat_Type: cobj as _PyFloat_Type
from C import PyBool_Type: cobj as _PyBool_Type from C import PyBool_Type: cobj as _PyBool_Type
from C import PyUnicode_Type: cobj as _PyUnicode_Type from C import PyUnicode_Type: cobj as _PyUnicode_Type
from C import PyBytes_Type: cobj as _PyBytes_Type
from C import PyComplex_Type: cobj as _PyComplex_Type from C import PyComplex_Type: cobj as _PyComplex_Type
from C import PyList_Type: cobj as _PyList_Type from C import PyList_Type: cobj as _PyList_Type
from C import PyDict_Type: cobj as _PyDict_Type from C import PyDict_Type: cobj as _PyDict_Type
@ -624,7 +616,6 @@ def init_handles_static():
global PyFloat_FromDouble global PyFloat_FromDouble
global PyBool_FromLong global PyBool_FromLong
global PyBytes_AsString global PyBytes_AsString
global PyBytes_Size
global PyList_New global PyList_New
global PyList_Size global PyList_Size
global PyList_GetItem global PyList_GetItem
@ -715,7 +706,6 @@ def init_handles_static():
global PyFloat_Type global PyFloat_Type
global PyBool_Type global PyBool_Type
global PyUnicode_Type global PyUnicode_Type
global PyBytes_Type
global PyComplex_Type global PyComplex_Type
global PyList_Type global PyList_Type
global PyDict_Type global PyDict_Type
@ -760,7 +750,6 @@ def init_handles_static():
PyFloat_FromDouble = _PyFloat_FromDouble PyFloat_FromDouble = _PyFloat_FromDouble
PyBool_FromLong = _PyBool_FromLong PyBool_FromLong = _PyBool_FromLong
PyBytes_AsString = _PyBytes_AsString PyBytes_AsString = _PyBytes_AsString
PyBytes_Size = _PyBytes_Size
PyList_New = _PyList_New PyList_New = _PyList_New
PyList_Size = _PyList_Size PyList_Size = _PyList_Size
PyList_GetItem = _PyList_GetItem PyList_GetItem = _PyList_GetItem
@ -851,7 +840,6 @@ def init_handles_static():
PyFloat_Type = __ptr__(_PyFloat_Type).as_byte() PyFloat_Type = __ptr__(_PyFloat_Type).as_byte()
PyBool_Type = __ptr__(_PyBool_Type).as_byte() PyBool_Type = __ptr__(_PyBool_Type).as_byte()
PyUnicode_Type = __ptr__(_PyUnicode_Type).as_byte() PyUnicode_Type = __ptr__(_PyUnicode_Type).as_byte()
PyBytes_Type = __ptr__(_PyBytes_Type).as_byte()
PyComplex_Type = __ptr__(_PyComplex_Type).as_byte() PyComplex_Type = __ptr__(_PyComplex_Type).as_byte()
PyList_Type = __ptr__(_PyList_Type).as_byte() PyList_Type = __ptr__(_PyList_Type).as_byte()
PyDict_Type = __ptr__(_PyDict_Type).as_byte() PyDict_Type = __ptr__(_PyDict_Type).as_byte()
@ -1186,7 +1174,7 @@ class pyobj:
return pyobj.to_str(self.p, errors, empty) return pyobj.to_str(self.p, errors, empty)
def to_str(p: cobj, errors: str, empty: str = "") -> str: def to_str(p: cobj, errors: str, empty: str = "") -> str:
obj = PyUnicode_AsEncodedString(p, "utf-8".ptr, errors.c_str() if errors else "".ptr) obj = PyUnicode_AsEncodedString(p, "utf-8".c_str(), errors.c_str())
if obj == cobj(): if obj == cobj():
return empty return empty
bts = PyBytes_AsString(obj) bts = PyBytes_AsString(obj)
@ -1304,11 +1292,8 @@ class _PyObject_Struct:
def _conversion_error(name: Static[str]): def _conversion_error(name: Static[str]):
raise PyError("conversion error: Python object did not have type '" + name + "'") raise PyError("conversion error: Python object did not have type '" + name + "'")
def _get_type(o: cobj):
return Ptr[_PyObject_Struct](o)[0].pytype
def _ensure_type(o: cobj, t: cobj, name: Static[str]): def _ensure_type(o: cobj, t: cobj, name: Static[str]):
if _get_type(o) != t: if Ptr[_PyObject_Struct](o)[0].pytype != t:
_conversion_error(name) _conversion_error(name)
@ -1365,14 +1350,7 @@ class str:
return pyobj.exc_wrap(PyUnicode_DecodeFSDefaultAndSize(self.ptr, self.len)) return pyobj.exc_wrap(PyUnicode_DecodeFSDefaultAndSize(self.ptr, self.len))
def __from_py__(s: cobj) -> str: def __from_py__(s: cobj) -> str:
if _get_type(s) == PyBytes_Type: return pyobj.exc_wrap(pyobj.to_str(s, "strict"))
n = PyBytes_Size(s)
p0 = PyBytes_AsString(s)
p1 = cobj(n)
str.memcpy(p1, p0, n)
return str(p1, n)
else:
return pyobj.exc_wrap(pyobj.to_str(s, "strict"))
@extend @extend
class complex: class complex:

View File

@ -120,19 +120,17 @@ class PyArrayDescr:
kind: u8 kind: u8
type: u8 type: u8
byteorder: u8 byteorder: u8
former_flags: u8 flags: u8
type_num: i32 type_num: i32
flags: u64 elsize: i32
elsize: int alignment: i32
alignment: int
metadata: cobj
hash: int
reserved0: cobj
reserved1: cobj
subarray: cobj subarray: cobj
fields: cobj fields: cobj
names: cobj names: cobj
f: cobj
metadata: cobj
c_metadata: cobj c_metadata: cobj
hash: int
@tuple @tuple
class PyArrayObject: class PyArrayObject:
@ -151,18 +149,21 @@ NPY_ARRAY_F_CONTIGUOUS: Static[int] = 2
PyArray_Type = cobj() PyArray_Type = cobj()
PyArray_New = Function[[cobj, i32, cobj, i32, cobj, cobj, i32, i32, cobj], cobj](cobj()) PyArray_New = Function[[cobj, i32, cobj, i32, cobj, cobj, i32, i32, cobj], cobj](cobj())
PyArray_NewFromDescr = Function[[cobj, cobj, i32, cobj, cobj, cobj, i32, cobj], cobj](cobj())
PyArray_DescrNewFromType = Function[[i32], cobj](cobj())
def _pyobj_type(p: cobj): def _pyobj_type(p: cobj):
return Ptr[PyObject](p)[0].typptr return Ptr[PyObject](p)[0].typptr
def _set_datetime_descr(a: Ptr[PyArrayObject], T: type):
p = Ptr[PyArray_DatetimeDTypeMetaData](a[0].descr[0].c_metadata)
meta = PyArray_DatetimeMetaData(i32(T._code()), i32(T.num))
p[0] = PyArray_DatetimeDTypeMetaData(p[0].base, meta)
def _setup_numpy_bridge(): def _setup_numpy_bridge():
import python import python
from internal.python import PyImport_ImportModule, PyObject_GetAttrString, PyCapsule_Type, PyCapsule_GetPointer from internal.python import PyImport_ImportModule, PyObject_GetAttrString, PyCapsule_Type, PyCapsule_GetPointer
global PyArray_Type, PyArray_New, PyArray_NewFromDescr, PyArray_DescrNewFromType global PyArray_Type, PyArray_New
module = PyImport_ImportModule("numpy._core._multiarray_umath".ptr) module = PyImport_ImportModule("numpy.core._multiarray_umath".ptr)
if not module: if not module:
raise RuntimeError("Failed to import 'numpy.core._multiarray_umath'") raise RuntimeError("Failed to import 'numpy.core._multiarray_umath'")
@ -176,8 +177,6 @@ def _setup_numpy_bridge():
# https://github.com/numpy/numpy/blob/main/numpy/_core/code_generators/numpy_api.py # https://github.com/numpy/numpy/blob/main/numpy/_core/code_generators/numpy_api.py
PyArray_Type = api[2] PyArray_Type = api[2]
PyArray_New = Function[[cobj, i32, cobj, i32, cobj, cobj, i32, i32, cobj], cobj](api[93]) PyArray_New = Function[[cobj, i32, cobj, i32, cobj, cobj, i32, i32, cobj], cobj](api[93])
PyArray_NewFromDescr = Function[[cobj, cobj, i32, cobj, cobj, cobj, i32, cobj], cobj](api[94])
PyArray_DescrNewFromType = Function[[i32], cobj](api[96])
_setup_numpy_bridge() _setup_numpy_bridge()
@ -186,19 +185,8 @@ class ndarray:
def __to_py__(self): def __to_py__(self):
dims = self.shape dims = self.shape
code = _type_code(self.dtype) code = _type_code(self.dtype)
arr = PyArray_New(PyArray_Type, i32(self.ndim), __ptr__(dims).as_byte(),
if isinstance(self.dtype, datetime64) or isinstance(self.dtype, timedelta64): i32(code), cobj(), cobj(), i32(0), i32(0), cobj())
descr = Ptr[PyArrayDescr](PyArray_DescrNewFromType(i32(code)))
p = Ptr[PyArray_DatetimeDTypeMetaData](descr[0].c_metadata)
meta = PyArray_DatetimeMetaData(i32(self.dtype._code()), i32(self.dtype.num))
p[0] = PyArray_DatetimeDTypeMetaData(p[0].base, meta)
arr = PyArray_NewFromDescr(PyArray_Type, descr.as_byte(), i32(self.ndim),
__ptr__(dims).as_byte(), cobj(), cobj(), i32(0),
cobj())
else:
arr = PyArray_New(PyArray_Type, i32(self.ndim), __ptr__(dims).as_byte(),
i32(code), cobj(), cobj(), i32(0), i32(0), cobj())
arr_ptr = Ptr[PyArrayObject](arr) arr_ptr = Ptr[PyArrayObject](arr)
data = arr_ptr[0].data data = arr_ptr[0].data
@ -222,6 +210,9 @@ class ndarray:
p[k] = e p[k] = e
k += 1 k += 1
if isinstance(self.dtype, datetime64) or isinstance(self.dtype, timedelta64):
_set_datetime_descr(arr_ptr, self.dtype)
return arr return arr
def _from_py(a: cobj, copy: bool): def _from_py(a: cobj, copy: bool):
@ -254,7 +245,7 @@ class ndarray:
k = 0 k = 0
for idx in util.multirange(shape): for idx in util.multirange(shape):
off = 0 off = 0
for i in staticrange(ndim): for i in range(ndim):
off += idx[i] * strides[i] off += idx[i] * strides[i]
e = Ptr[cobj](arr_data + off)[0] e = Ptr[cobj](arr_data + off)[0]
if hasattr(dtype, "__from_py__"): if hasattr(dtype, "__from_py__"):
@ -272,7 +263,7 @@ class ndarray:
k = 0 k = 0
for idx in util.multirange(shape): for idx in util.multirange(shape):
off = 0 off = 0
for i in staticrange(ndim): for i in range(ndim):
off += idx[i] * strides[i] off += idx[i] * strides[i]
e = Ptr[dtype](arr_data + off)[0] e = Ptr[dtype](arr_data + off)[0]
data[k] = e data[k] = e

View File

@ -181,16 +181,3 @@ def test_ndarray():
assert np.datetime_data(y.dtype) == ('s', 2) assert np.datetime_data(y.dtype) == ('s', 2)
test_ndarray() test_ndarray()
@codon.jit
def e(x=2, y=99):
return 2*x + y
def test_arg_order():
assert e(1, 2) == 4
assert e(1) == 101
assert e(y=10, x=1) == 12
assert e(x=1) == 101
assert e() == 103
test_arg_order()

View File

@ -88,7 +88,6 @@ def test_codon_extensions(m):
assert m.f4(a=2.2) == (2.2, 2.22) assert m.f4(a=2.2) == (2.2, 2.22)
assert m.f4(b=3.3) == (1.11, 3.3) assert m.f4(b=3.3) == (1.11, 3.3)
assert m.f4('foo') == ('foo', 'foo') assert m.f4('foo') == ('foo', 'foo')
assert m.f4(b'foo') == ('foo', 'foo')
assert m.f4({1}) == {1} assert m.f4({1}) == {1}
assert m.f5() is None assert m.f5() is None
assert equal(m.f6(1.9, 't'), 1.9, 1.9, 't') assert equal(m.f6(1.9, 't'), 1.9, 1.9, 't')