mirrors
/
codon
mirror of https://github.com/exaloop/codon.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

JIT (#6) 

* Add JIT engine

* More engine updates

* Fix takeModule()

* Parser JIT support [wip]

* Fix class definition

* Parser JIT support [wip]

* Parser JIT support [wip]

* Fix LLVM context conflict

* Parser JIT support [wip]

* Fix JIT engine

* Parser JIT support [wip]

* Doc fix

* JIT fix

* Fix JIT exceptions

* Refactor

* Add JIT status codes

* Add compiler class

* Better logging

* Better logging

* Update .gitignore

* Add Jupyter Xeus support

* Update JIT

* Remove print

* Update errors

* Fix assert

* Fix asserts

* Update docgen signature

* Update file system interface

* Update plugin errors

* Use numeric_limits

* JIT refactor [wip]

* Set JIT flag on LLVMVisitor

* Update module opt

* JIT imports

* First Jupyter integration

* Update JIT API to return outputs as string

* Capture runtime exception output

* Fix libbacktrace build

* Initial Jupyter support

* Format

* Fix print

* Support run-mode backtraces

* Fix multithreaded backtrace

* Update backtraces

* Upgrade OpenMP

* Add libunwind

* Fix build

* Fix build

* Fix build

* Fix build

* Fix OpenMP & tests

* Use libbacktrace instead of libunwind

* Add debug listener

* Remove unused include

* Remove unused class

* Fix backtraces

* Update backtrace config

* Fix debug info generation

* Refactor backtraces

* Fix ASAN flag

* Fix JIT

* Fix JIT backtraces

* Fix JIT backtrace

* Fix Jupyter, fix xeus build flags

* Fix JIT output capture

* Fix Jupyter

* Fix Jupyter Python support

* Add __repr_pretty__ support

* Update JIT output capturing

* Better backtrace method names

* Support plugins in JIT mode

Co-authored-by: Ibrahim Numanagić <ibrahimpasa@gmail.com>
pull/9/head
A. R. Shajii 2021-11-30 11:50:28 -05:00 committed by GitHub
parent 81597d22fa
commit 166e1ad455
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
79 changed files with 2766 additions and 1388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -16,6 +16,7 @@
*.pyc
build/
build_*/
extra/jupyter/build/
# Packages #
############
@ -50,3 +51,6 @@ Thumbs.db
.idea
.mypy_cache
.vscode
extra/jupyter/share/jupyter/kernels/codon/kernel.json
scratch.seq

55
CMakeLists.txt
View File

@ -7,17 +7,20 @@ configure_file("${PROJECT_SOURCE_DIR}/cmake/config.h.in"
configure_file("${PROJECT_SOURCE_DIR}/cmake/config.py.in"
"${PROJECT_SOURCE_DIR}/docs/sphinx/config/config.py")
option(CODON_JUPYTER "build Codon Jupyter server" OFF)
if(CODON_JUPYTER)
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/extra/jupyter/share/jupyter/kernels/codon/kernel.json.in"
"${CMAKE_CURRENT_SOURCE_DIR}/extra/jupyter/share/jupyter/kernels/codon/kernel.json"
)
endif()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility-inlines-hidden -pedantic -Wall -Wno-return-type-c-linkage -Wno-gnu-zero-variadic-macro-arguments")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility-inlines-hidden -pedantic -Wno-return-type-c-linkage -Wno-gnu-zero-variadic-macro-arguments")
set(CMAKE_CXX_FLAGS_DEBUG "-g -fno-limit-debug-info")
set(CMAKE_CXX_FLAGS_RELEASE "-O3")
include_directories(.)
if(ASAN)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
set(CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
endif()
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
find_package(LLVM REQUIRED)
@ -70,14 +73,23 @@ else()
$<TARGET_FILE:liblzma>
-Wl,--no-whole-archive)
endif()
if(ASAN)
target_compile_options(codonrt PRIVATE "-fno-omit-frame-pointer" "-fsanitize=address")
target_link_libraries(codonrt PRIVATE "-fno-omit-frame-pointer" "-fsanitize=address")
endif()
add_custom_command(TARGET codonrt POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:omp> ${CMAKE_BINARY_DIR})
# Codon compiler library
include_directories(${LLVM_INCLUDE_DIRS})
add_definitions(${LLVM_DEFINITIONS})
set(CODON_HPPFILES
codon/compiler/compiler.h
codon/compiler/debug_listener.h
codon/compiler/engine.h
codon/compiler/error.h
codon/compiler/jit.h
codon/compiler/memory_manager.h
codon/dsl/dsl.h
codon/dsl/plugins.h
codon/parser/ast.h
@ -89,7 +101,6 @@ set(CODON_HPPFILES
codon/parser/ctx.h
codon/parser/peg/peg.h
codon/parser/peg/rules.h
codon/parser/parser.h
codon/parser/visitors/doc/doc.h
codon/parser/visitors/format/format.h
codon/parser/visitors/simplify/simplify.h
@ -122,7 +133,6 @@ set(CODON_HPPFILES
codon/sir/llvm/coro/Coroutines.h
codon/sir/llvm/llvisitor.h
codon/sir/llvm/llvm.h
codon/sir/llvm/memory_manager.h
codon/sir/llvm/optimize.h
codon/sir/module.h
codon/sir/sir.h
@ -198,8 +208,15 @@ set(CODON_HPPFILES
codon/util/toml++/toml_default_formatter.h
codon/util/toml++/toml_node.h
codon/util/toml++/toml_parser.hpp
codon/util/toml++/toml_utf8_streams.h)
codon/util/toml++/toml_utf8_streams.h
extra/jupyter/src/codon.h)
set(CODON_CPPFILES
codon/compiler/compiler.cpp
codon/compiler/debug_listener.cpp
codon/compiler/engine.cpp
codon/compiler/error.cpp
codon/compiler/jit.cpp
codon/compiler/memory_manager.cpp
codon/dsl/plugins.cpp
codon/parser/ast/expr.cpp
codon/parser/ast/stmt.cpp
@ -207,7 +224,6 @@ set(CODON_CPPFILES
codon/parser/cache.cpp
codon/parser/common.cpp
codon/parser/peg/peg.cpp
codon/parser/parser.cpp
codon/parser/visitors/doc/doc.cpp
codon/parser/visitors/format/format.cpp
codon/parser/visitors/simplify/simplify.cpp
@ -242,7 +258,6 @@ set(CODON_CPPFILES
codon/sir/llvm/coro/CoroSplit.cpp
codon/sir/llvm/coro/Coroutines.cpp
codon/sir/llvm/llvisitor.cpp
codon/sir/llvm/memory_manager.cpp
codon/sir/llvm/optimize.cpp
codon/sir/module.cpp
codon/sir/transform/cleanup/canonical.cpp
@ -272,9 +287,19 @@ set(CODON_CPPFILES
codon/sir/value.cpp
codon/sir/var.cpp
codon/util/common.cpp
codon/util/fmt/format.cpp)
codon/util/fmt/format.cpp
extra/jupyter/src/codon.cpp)
add_library(codonc SHARED ${CODON_HPPFILES})
target_sources(codonc PRIVATE ${CODON_CPPFILES} codon_rules.cpp omp_rules.cpp)
if(CODON_JUPYTER)
add_compile_definitions(CODON_JUPYTER)
add_dependencies(codonc xeus-static nlohmann_json)
target_link_libraries(codonc PRIVATE xeus-static)
endif()
if(ASAN)
target_compile_options(codonc PRIVATE "-fno-omit-frame-pointer" "-fsanitize=address")
target_link_libraries(codonc PRIVATE "-fno-omit-frame-pointer" "-fsanitize=address")
endif()
if(CMAKE_BUILD_TYPE MATCHES Debug)
set_source_files_properties(codon_rules.cpp codon/parser/peg/peg.cpp PROPERTIES COMPILE_FLAGS "-O2")
endif()
@ -301,6 +326,7 @@ llvm_map_components_to_libnames(LLVM_LIBS
Remarks
ScalarOpts
Support
Symbolize
Target
TransformUtils
Vectorize
@ -328,7 +354,7 @@ add_dependencies(headers codonrt codonc)
# Codon command-line tool
add_executable(codon codon/app/main.cpp)
target_link_libraries(codon ${STATIC_LIBCPP} codonc Threads::Threads)
target_link_libraries(codon PUBLIC ${STATIC_LIBCPP} codonc Threads::Threads)
# Codon test
# Download and unpack googletest at configure time
@ -359,3 +385,4 @@ add_executable(codon_test ${CODON_TEST_CPPFILES})
target_include_directories(codon_test PRIVATE test/sir "${gc_SOURCE_DIR}/include")
target_link_libraries(codon_test codonc codonrt gtest_main)
target_compile_definitions(codon_test PRIVATE TEST_DIR="${CMAKE_CURRENT_SOURCE_DIR}/test")

3
cmake/backtrace-config.h.in
View File

@ -46,6 +46,9 @@
/* Define to 1 if you have the `lstat' function. */
#cmakedefine HAVE_LSTAT 1
/* Define to 1 if you have the <mach-o/dyld.h> header file. */
#cmakedefine HAVE_MACH_O_DYLD_H 1
/* Define to 1 if you have the <memory.h> header file. */
#cmakedefine HAVE_MEMORY_H 1

85
cmake/deps.cmake
View File

@ -66,7 +66,6 @@ if(bdwgc_ADDED)
endif()
CPMAddPackage(
NAME openmp
GITHUB_REPOSITORY "llvm-mirror/openmp"
VERSION 9.0
GIT_TAG release_90
@ -79,29 +78,10 @@ CPMAddPackage(
GIT_TAG d0f5e95a87a4d3e0a1ed6c069b5dae7cbab3ed2a
DOWNLOAD_ONLY YES)
if(backtrace_ADDED)
# https://go.googlesource.com/gollvm/+/refs/heads/master/cmake/modules/LibbacktraceUtils.cmake
set(BACKTRACE_ELF_SIZE 64)
set(HAVE_GETIPINFO 1)
set(BACKTRACE_USES_MALLOC 0)
set(BACKTRACE_SUPPORTS_THREADS 1)
set(BACKTRACE_SUPPORTS_DATA 1)
if(HAVE_SYNC_BOOL_COMPARE_AND_SWAP_4)
if(HAVE_SYNC_BOOL_COMPARE_AND_SWAP_8)
set(HAVE_SYNC_FUNCTIONS 1)
endif()
endif()
# Generate backtrace-supported.h based on the above.
configure_file(
${CMAKE_SOURCE_DIR}/cmake/backtrace-supported.h.in
${backtrace_SOURCE_DIR}/backtrace-supported.h)
configure_file(
${CMAKE_SOURCE_DIR}/cmake/backtrace-config.h.in
${backtrace_SOURCE_DIR}/config.h)
add_library(backtrace STATIC
set(backtrace_SOURCES
"${backtrace_SOURCE_DIR}/atomic.c"
"${backtrace_SOURCE_DIR}/backtrace.c"
"${backtrace_SOURCE_DIR}/dwarf.c"
"${backtrace_SOURCE_DIR}/elf.c"
"${backtrace_SOURCE_DIR}/fileline.c"
"${backtrace_SOURCE_DIR}/mmapio.c"
"${backtrace_SOURCE_DIR}/mmap.c"
@ -110,8 +90,71 @@ if(backtrace_ADDED)
"${backtrace_SOURCE_DIR}/simple.c"
"${backtrace_SOURCE_DIR}/sort.c"
"${backtrace_SOURCE_DIR}/state.c")
# https://go.googlesource.com/gollvm/+/refs/heads/master/cmake/modules/LibbacktraceUtils.cmake
set(BACKTRACE_SUPPORTED 1)
set(BACKTRACE_ELF_SIZE 64)
set(HAVE_GETIPINFO 1)
set(BACKTRACE_USES_MALLOC 1)
set(BACKTRACE_SUPPORTS_THREADS 1)
set(BACKTRACE_SUPPORTS_DATA 1)
set(HAVE_SYNC_FUNCTIONS 1)
if(APPLE)
set(HAVE_MACH_O_DYLD_H 1)
list(APPEND backtrace_SOURCES "${backtrace_SOURCE_DIR}/macho.c")
else()
set(HAVE_MACH_O_DYLD_H 0)
list(APPEND backtrace_SOURCES "${backtrace_SOURCE_DIR}/elf.c")
endif()
# Generate backtrace-supported.h based on the above.
configure_file(
${CMAKE_SOURCE_DIR}/cmake/backtrace-supported.h.in
${backtrace_SOURCE_DIR}/backtrace-supported.h)
configure_file(
${CMAKE_SOURCE_DIR}/cmake/backtrace-config.h.in
${backtrace_SOURCE_DIR}/config.h)
add_library(backtrace STATIC ${backtrace_SOURCES})
target_include_directories(backtrace BEFORE PRIVATE "${backtrace_SOURCE_DIR}")
set_target_properties(backtrace PROPERTIES
COMPILE_FLAGS "-funwind-tables -D_GNU_SOURCE"
POSITION_INDEPENDENT_CODE ON)
endif()
if(CODON_JUPYTER)
CPMAddPackage(
NAME libzmq
VERSION 4.3.4
URL https://github.com/zeromq/libzmq/releases/download/v4.3.4/zeromq-4.3.4.tar.gz
OPTIONS "WITH_PERF_TOOL OFF"
"ZMQ_BUILD_TESTS OFF"
"ENABLE_CPACK OFF"
"BUILD_SHARED ON"
"WITH_LIBSODIUM OFF")
CPMAddPackage(
NAME cppzmq
URL https://github.com/zeromq/cppzmq/archive/refs/tags/v4.8.1.tar.gz
VERSION 4.8.1
OPTIONS "CPPZMQ_BUILD_TESTS OFF")
CPMAddPackage(
NAME xtl
GITHUB_REPOSITORY "xtensor-stack/xtl"
VERSION 0.7.3
GIT_TAG 0.7.3
OPTIONS "BUILD_TESTS OFF")
CPMAddPackage(
NAME json
GITHUB_REPOSITORY "nlohmann/json"
VERSION 3.10.4)
CPMAddPackage(
NAME xeus
GITHUB_REPOSITORY "jupyter-xeus/xeus"
VERSION 2.2.0
GIT_TAG 2.2.0
PATCH_COMMAND sed -i bak "s/-Wunused-parameter -Wextra -Wreorder//g" CMakeLists.txt
OPTIONS "BUILD_EXAMPLES OFF"
"XEUS_BUILD_SHARED_LIBS OFF"
"XEUS_STATIC_DEPENDENCIES ON")
if (xeus_ADDED)
install(TARGETS nlohmann_json EXPORT xeus-targets)
endif()
endif()

244
codon/app/main.cpp
View File

@ -1,17 +1,18 @@
#include "codon/dsl/plugins.h"
#include "codon/parser/parser.h"
#include "codon/sir/llvm/llvisitor.h"
#include "codon/sir/transform/manager.h"
#include "codon/sir/transform/pass.h"
#include "codon/util/common.h"
#include "llvm/Support/CommandLine.h"
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include "codon/compiler/compiler.h"
#include "codon/compiler/error.h"
#include "codon/compiler/jit.h"
#include "codon/util/common.h"
#include "llvm/Support/CommandLine.h"
namespace {
void versMsg(llvm::raw_ostream &out) {
out << CODON_VERSION_MAJOR << "." << CODON_VERSION_MINOR << "." << CODON_VERSION_PATCH
@ -46,23 +47,41 @@ std::string makeOutputFilename(const std::string &filename,
return filename + extension;
}
void display(const codon::error::ParserErrorInfo &e) {
for (auto &msg : e) {
codon::compilationError(msg.getMessage(), msg.getFile(), msg.getLine(),
msg.getColumn(),
/*terminate=*/false);
}
}
enum BuildKind { LLVM, Bitcode, Object, Executable, Detect };
enum OptMode { Debug, Release };
struct ProcessResult {
std::unique_ptr<codon::ir::transform::PassManager> pm;
std::unique_ptr<codon::PluginManager> plm;
std::unique_ptr<codon::ir::LLVMVisitor> visitor;
std::string input;
};
} // namespace
int docMode(const std::vector<const char *> &args, const std::string &argv0) {
llvm::cl::ParseCommandLineOptions(args.size(), args.data());
codon::generateDocstr(argv0);
std::vector<std::string> files;
for (std::string line; std::getline(std::cin, line);)
files.push_back(line);
auto compiler = std::make_unique<codon::Compiler>(args[0]);
bool failed = false;
auto result = compiler->docgen(files);
llvm::handleAllErrors(result.takeError(),
[&failed](const codon::error::ParserErrorInfo &e) {
display(e);
failed = true;
});
if (failed)
return EXIT_FAILURE;
fmt::print("{}\n", *result);
return EXIT_SUCCESS;
}
ProcessResult processSource(const std::vector<const char *> &args) {
std::unique_ptr<codon::Compiler> processSource(const std::vector<const char *> &args,
bool standalone) {
llvm::cl::opt<std::string> input(llvm::cl::Positional, llvm::cl::desc("<input file>"),
llvm::cl::init("-"));
llvm::cl::opt<OptMode> optMode(
@ -80,8 +99,12 @@ ProcessResult processSource(const std::vector<const char *> &args) {
"disable-opt", llvm::cl::desc("Disable the specified IR optimization"));
llvm::cl::list<std::string> plugins("plugin",
llvm::cl::desc("Load specified plugin"));
llvm::cl::opt<std::string> log("log", llvm::cl::desc("Enable given log streams"));
llvm::cl::ParseCommandLineOptions(args.size(), args.data());
codon::getLogger().parse(log);
if (auto *d = getenv("CODON_DEBUG"))
codon::getLogger().parse(std::string(d));
auto &exts = supportedExtensions();
if (input != "-" && std::find_if(exts.begin(), exts.end(), [&](auto &ext) {
@ -110,54 +133,37 @@ ProcessResult processSource(const std::vector<const char *> &args) {
}
const bool isDebug = (optMode == OptMode::Debug);
auto t = std::chrono::high_resolution_clock::now();
std::vector<std::string> disabledOptsVec(disabledOpts);
auto pm =
std::make_unique<codon::ir::transform::PassManager>(isDebug, disabledOptsVec);
auto plm = std::make_unique<codon::PluginManager>();
auto compiler = std::make_unique<codon::Compiler>(args[0], isDebug, disabledOptsVec);
compiler->getLLVMVisitor()->setStandalone(standalone);
LOG_TIME("[T] ir-setup = {:.1f}",
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - t)
.count() /
1000.0);
// load other plugins
// load plugins
for (const auto &plugin : plugins) {
std::string errMsg;
if (!plm->load(plugin, &errMsg))
codon::compilationError(errMsg);
bool failed = false;
llvm::handleAllErrors(
compiler->load(plugin), [&failed](const codon::error::PluginErrorInfo &e) {
codon::compilationError(e.getMessage(), /*file=*/"", /*line=*/0, /*col=*/0,
/*terminate=*/false);
failed = true;
});
if (failed)
return {};
}
// add all IR passes
for (const auto *plugin : *plm) {
plugin->dsl->addIRPasses(pm.get(), isDebug);
bool failed = false;
llvm::handleAllErrors(compiler->parseFile(input, /*isTest=*/0, defmap),
[&failed](const codon::error::ParserErrorInfo &e) {
display(e);
failed = true;
});
if (failed)
return {};
{
TIME("compile");
llvm::cantFail(compiler->compile());
}
t = std::chrono::high_resolution_clock::now();
auto *module = codon::parse(args[0], input.c_str(), /*code=*/"", /*isCode=*/false,
/*isTest=*/false, /*startLine=*/0, defmap, plm.get());
if (!module)
return {{}, {}};
pm->run(module);
LOG_TIME("[T] ir-opt = {:.1f}", std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - t)
.count() /
1000.0);
t = std::chrono::high_resolution_clock::now();
auto visitor = std::make_unique<codon::ir::LLVMVisitor>(isDebug);
visitor->setPluginManager(plm.get());
visitor->visit(module);
LOG_TIME("[T] ir-visitor = {:.1f}",
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - t)
.count() /
1000.0);
if (_dbg_level)
visitor->dump();
return {std::move(pm), std::move(plm), std::move(visitor), input};
return compiler;
}
int runMode(const std::vector<const char *> &args) {
@ -165,19 +171,72 @@ int runMode(const std::vector<const char *> &args) {
"l", llvm::cl::desc("Load and link the specified library"));
llvm::cl::list<std::string> seqArgs(llvm::cl::ConsumeAfter,
llvm::cl::desc("<program arguments>..."));
auto start_t = std::chrono::high_resolution_clock::now();
auto result = processSource(args);
if (!result.visitor)
auto compiler = processSource(args, /*standalone=*/false);
if (!compiler)
return EXIT_FAILURE;
std::vector<std::string> libsVec(libs);
std::vector<std::string> argsVec(seqArgs);
argsVec.insert(argsVec.begin(), result.input);
LOG_USER("compiler took: {:.2f} seconds",
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - start_t)
.count() /
1000.0);
result.visitor->run(argsVec, libsVec);
argsVec.insert(argsVec.begin(), compiler->getInput());
compiler->getLLVMVisitor()->run(argsVec, libsVec);
return EXIT_SUCCESS;
}
namespace {
std::string jitExec(codon::jit::JIT *jit, const std::string &code) {
auto result = jit->exec(code);
if (auto err = result.takeError()) {
std::string output;
llvm::handleAllErrors(
std::move(err), [](const codon::error::ParserErrorInfo &e) { display(e); },
[&output](const codon::error::RuntimeErrorInfo &e) {
std::stringstream buf;
buf << e.getOutput();
buf << "\n\033[1mBacktrace:\033[0m\n";
for (const auto &line : e.getBacktrace()) {
buf << " " << line << "\n";
}
output = buf.str();
});
return output;
}
return *result;
}
} // namespace
int jitMode(const std::vector<const char *> &args) {
llvm::cl::list<std::string> plugins("plugin",
llvm::cl::desc("Load specified plugin"));
llvm::cl::ParseCommandLineOptions(args.size(), args.data());
codon::jit::JIT jit(args[0]);
// load plugins
for (const auto &plugin : plugins) {
bool failed = false;
llvm::handleAllErrors(jit.getCompiler()->load(plugin),
[&failed](const codon::error::PluginErrorInfo &e) {
codon::compilationError(e.getMessage(), /*file=*/"",
/*line=*/0, /*col=*/0,
/*terminate=*/false);
failed = true;
});
if (failed)
return EXIT_FAILURE;
}
llvm::cantFail(jit.init());
fmt::print(">>> Codon JIT v{} <<<\n", CODON_VERSION);
std::string code;
for (std::string line; std::getline(std::cin, line);) {
if (line != "#%%") {
code += line + "\n";
} else {
fmt::print("{}\n\n[done]\n\n", jitExec(&jit, code));
code = "";
fflush(stdout);
}
}
if (!code.empty())
fmt::print("{}\n\n[done]\n\n", jitExec(&jit, code));
return EXIT_SUCCESS;
}
@ -199,12 +258,12 @@ int buildMode(const std::vector<const char *> &args) {
"Write compiled output to specified file. Supported extensions: "
"none (executable), .o (object file), .ll (LLVM IR), .bc (LLVM bitcode)"));
auto result = processSource(args);
if (!result.visitor)
auto compiler = processSource(args, /*standalone=*/true);
if (!compiler)
return EXIT_FAILURE;
std::vector<std::string> libsVec(libs);
if (output.empty() && result.input == "-")
if (output.empty() && compiler->getInput() == "-")
codon::compilationError("output file must be specified when reading from stdin");
std::string extension;
switch (buildKind) {
@ -222,33 +281,50 @@ int buildMode(const std::vector<const char *> &args) {
extension = "";
break;
default:
assert(0);
seqassert(0, "unknown build kind");
}
const std::string filename =
output.empty() ? makeOutputFilename(result.input, extension) : output;
output.empty() ? makeOutputFilename(compiler->getInput(), extension) : output;
switch (buildKind) {
case BuildKind::LLVM:
result.visitor->writeToLLFile(filename);
compiler->getLLVMVisitor()->writeToLLFile(filename);
break;
case BuildKind::Bitcode:
result.visitor->writeToBitcodeFile(filename);
compiler->getLLVMVisitor()->writeToBitcodeFile(filename);
break;
case BuildKind::Object:
result.visitor->writeToObjectFile(filename);
compiler->getLLVMVisitor()->writeToObjectFile(filename);
break;
case BuildKind::Executable:
result.visitor->writeToExecutable(filename, libsVec);
compiler->getLLVMVisitor()->writeToExecutable(filename, libsVec);
break;
case BuildKind::Detect:
result.visitor->compile(filename, libsVec);
compiler->getLLVMVisitor()->compile(filename, libsVec);
break;
default:
assert(0);
seqassert(0, "unknown build kind");
}
return EXIT_SUCCESS;
}
#ifdef CODON_JUPYTER
namespace codon {
int startJupyterKernel(const std::string &argv0, const std::string &configPath);
}
#endif
int jupyterMode(const std::vector<const char *> &args) {
#ifdef CODON_JUPYTER
int code = codon::startJupyterKernel(args[0], args.size() > 1 ? std::string(args[1])
: "connection.json");
return code;
#else
fmt::print("Jupyter support not included. Please recompile with "
"-DCODON_JUPYTER.");
return EXIT_FAILURE;
#endif
}
void showCommandsAndExit() {
codon::compilationError("Available commands: seqc <run|build|doc>");
}
@ -290,5 +366,13 @@ int main(int argc, const char **argv) {
args[0] = argv0.data();
return docMode(args, oldArgv0);
}
if (mode == "jit") {
args[0] = argv0.data();
return jitMode(args);
}
if (mode == "jupyter") {
args[0] = argv0.data();
return jupyterMode(args);
}
return otherMode({argv, argv + argc});
}

116
codon/compiler/compiler.cpp 100644
View File

@ -0,0 +1,116 @@
#include "compiler.h"
#include <filesystem>
#include "codon/parser/cache.h"
#include "codon/parser/peg/peg.h"
#include "codon/parser/visitors/doc/doc.h"
#include "codon/parser/visitors/format/format.h"
#include "codon/parser/visitors/simplify/simplify.h"
#include "codon/parser/visitors/translate/translate.h"
#include "codon/parser/visitors/typecheck/typecheck.h"
namespace codon {
Compiler::Compiler(const std::string &argv0, bool debug,
const std::vector<std::string> &disabledPasses, bool isTest)
: argv0(argv0), debug(debug), input(), plm(std::make_unique<PluginManager>()),
cache(std::make_unique<ast::Cache>(argv0)),
module(std::make_unique<ir::Module>()),
pm(std::make_unique<ir::transform::PassManager>(debug && !isTest,
disabledPasses)),
llvisitor(std::make_unique<ir::LLVMVisitor>()) {
cache->module = module.get();
module->setCache(cache.get());
llvisitor->setDebug(debug);
llvisitor->setPluginManager(plm.get());
}
llvm::Error Compiler::load(const std::string &plugin) {
auto result = plm->load(plugin);
if (auto err = result.takeError())
return err;
auto *p = *result;
if (!p->info.stdlibPath.empty()) {
cache->pluginImportPaths.push_back(p->info.stdlibPath);
}
for (auto &kw : p->dsl->getExprKeywords()) {
cache->customExprStmts[kw.keyword] = kw.callback;
}
for (auto &kw : p->dsl->getBlockKeywords()) {
cache->customBlockStmts[kw.keyword] = {kw.hasExpr, kw.callback};
}
p->dsl->addIRPasses(pm.get(), debug);
return llvm::Error::success();
}
llvm::Error
Compiler::parse(bool isCode, const std::string &file, const std::string &code,
int startLine, int testFlags,
const std::unordered_map<std::string, std::string> &defines) {
input = file;
std::string abspath =
(file != "-") ? std::filesystem::absolute(std::filesystem::path(file)).string()
: file;
try {
Timer t1("parse");
ast::StmtPtr codeStmt = isCode
? ast::parseCode(cache.get(), abspath, code, startLine)
: ast::parseFile(cache.get(), abspath);
t1.log();
cache->module0 = file;
if (testFlags)
cache->testFlags = testFlags;
Timer t2("simplify");
auto transformed = ast::SimplifyVisitor::apply(cache.get(), std::move(codeStmt),
abspath, defines, (testFlags > 1));
t2.log();
Timer t3("typecheck");
auto typechecked =
ast::TypecheckVisitor::apply(cache.get(), std::move(transformed));
t3.log();
Timer t4("translate");
ast::TranslateVisitor::apply(cache.get(), std::move(typechecked));
t4.log();
} catch (const exc::ParserException &e) {
return llvm::make_error<error::ParserErrorInfo>(e);
}
module->setSrcInfo({abspath, 0, 0, 0});
return llvm::Error::success();
}
llvm::Error
Compiler::parseFile(const std::string &file, int testFlags,
const std::unordered_map<std::string, std::string> &defines) {
return parse(/*isCode=*/false, file, /*code=*/"", /*startLine=*/0, testFlags,
defines);
}
llvm::Error
Compiler::parseCode(const std::string &file, const std::string &code, int startLine,
int testFlags,
const std::unordered_map<std::string, std::string> &defines) {
return parse(/*isCode=*/true, file, code, startLine, testFlags, defines);
}
llvm::Error Compiler::compile() {
pm->run(module.get());
llvisitor->visit(module.get());
return llvm::Error::success();
}
llvm::Expected<std::string> Compiler::docgen(const std::vector<std::string> &files) {
try {
auto j = ast::DocVisitor::apply(argv0, files);
return j->toString();
} catch (exc::ParserException &e) {
return llvm::make_error<error::ParserErrorInfo>(e);
}
}
} // namespace codon

55
codon/compiler/compiler.h 100644
View File

@ -0,0 +1,55 @@
#pragma once
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "codon/compiler/error.h"
#include "codon/dsl/plugins.h"
#include "codon/parser/cache.h"
#include "codon/sir/llvm/llvisitor.h"
#include "codon/sir/module.h"
#include "codon/sir/transform/manager.h"
namespace codon {
class Compiler {
private:
std::string argv0;
bool debug;
std::string input;
std::unique_ptr<PluginManager> plm;
std::unique_ptr<ast::Cache> cache;
std::unique_ptr<ir::Module> module;
std::unique_ptr<ir::transform::PassManager> pm;
std::unique_ptr<ir::LLVMVisitor> llvisitor;
llvm::Error parse(bool isCode, const std::string &file, const std::string &code,
int startLine, int testFlags,
const std::unordered_map<std::string, std::string> &defines);
public:
Compiler(const std::string &argv0, bool debug = false,
const std::vector<std::string> &disabledPasses = {}, bool isTest = false);
std::string getInput() const { return input; }
PluginManager *getPluginManager() const { return plm.get(); }
ast::Cache *getCache() const { return cache.get(); }
ir::Module *getModule() const { return module.get(); }
ir::transform::PassManager *getPassManager() const { return pm.get(); }
ir::LLVMVisitor *getLLVMVisitor() const { return llvisitor.get(); }
llvm::Error load(const std::string &plugin);
llvm::Error
parseFile(const std::string &file, int testFlags = 0,
const std::unordered_map<std::string, std::string> &defines = {});
llvm::Error
parseCode(const std::string &file, const std::string &code, int startLine = 0,
int testFlags = 0,
const std::unordered_map<std::string, std::string> &defines = {});
llvm::Error compile();
llvm::Expected<std::string> docgen(const std::vector<std::string> &files);
};
} // namespace codon

73
codon/compiler/debug_listener.cpp 100644
View File

@ -0,0 +1,73 @@
#include "debug_listener.h"
#include <algorithm>
#include <sstream>
#include "codon/runtime/lib.h"
namespace codon {
void DebugListener::notifyObjectLoaded(ObjectKey key,
const llvm::object::ObjectFile &obj,
const llvm::RuntimeDyld::LoadedObjectInfo &L) {
uintptr_t start = 0, stop = 0;
for (const auto &sec : obj.sections()) {
if (sec.isText()) {
start = L.getSectionLoadAddress(sec);
stop = start + sec.getSize();
break;
}
}
auto buf = llvm::MemoryBuffer::getMemBufferCopy(obj.getData(), obj.getFileName());
auto newObj = llvm::cantFail(
llvm::object::ObjectFile::createObjectFile(buf->getMemBufferRef()));
objects.emplace_back(key, std::move(newObj), std::move(buf), start, stop);
}
void DebugListener::notifyFreeingObject(ObjectKey key) {
objects.erase(
std::remove_if(objects.begin(), objects.end(),
[key](const ObjectInfo &o) { return key == o.getKey(); }),
objects.end());
}
llvm::Expected<llvm::DILineInfo> DebugListener::symbolize(uintptr_t pc) {
for (const auto &o : objects) {
if (o.contains(pc)) {
llvm::symbolize::LLVMSymbolizer::Options opt;
opt.PrintFunctions = llvm::DILineInfoSpecifier::FunctionNameKind::ShortName;
opt.PathStyle = llvm::DILineInfoSpecifier::FileLineInfoKind::BaseNameOnly;
llvm::symbolize::LLVMSymbolizer sym(opt);
return sym.symbolizeCode(
o.getObject(),
{pc - o.getStart(), llvm::object::SectionedAddress::UndefSection});
}
}
return llvm::DILineInfo();
}
llvm::Expected<std::string> DebugListener::getPrettyBacktrace(uintptr_t pc) {
auto invalid = [](const std::string &name) { return name == "<invalid>"; };
auto src = symbolize(pc);
if (auto err = src.takeError())
return std::move(err);
if (invalid(src->FunctionName) || invalid(src->FileName))
return "";
return makeBacktraceFrameString(pc, src->FunctionName, src->FileName, src->Line,
src->Column);
}
std::string DebugListener::getPrettyBacktrace(const std::vector<uintptr_t> &backtrace) {
std::ostringstream buf;
buf << "\033[1mBacktrace:\033[0m\n";
for (auto pc : backtrace) {
auto line = getPrettyBacktrace(pc);
if (!line)
break;
if (!line->empty())
buf << " " << *line << "\n";
}
return buf.str();
}
} // namespace codon

49
codon/compiler/debug_listener.h 100644
View File

@ -0,0 +1,49 @@
#pragma once
#include <memory>
#include <vector>
#include "codon/sir/llvm/llvm.h"
namespace codon {
class DebugListener : public llvm::JITEventListener {
public:
class ObjectInfo {
private:
ObjectKey key;
std::unique_ptr<llvm::object::ObjectFile> object;
std::unique_ptr<llvm::MemoryBuffer> buffer;
uintptr_t start;
uintptr_t stop;
public:
ObjectInfo(ObjectKey key, std::unique_ptr<llvm::object::ObjectFile> object,
std::unique_ptr<llvm::MemoryBuffer> buffer, uintptr_t start,
uintptr_t stop)
: key(key), object(std::move(object)), buffer(std::move(buffer)), start(start),
stop(stop) {}
ObjectKey getKey() const { return key; }
const llvm::object::ObjectFile &getObject() const { return *object; }
uintptr_t getStart() const { return start; }
uintptr_t getStop() const { return stop; }
bool contains(uintptr_t pc) const { return start <= pc && pc < stop; }
};
private:
std::vector<ObjectInfo> objects;
void notifyObjectLoaded(ObjectKey key, const llvm::object::ObjectFile &obj,
const llvm::RuntimeDyld::LoadedObjectInfo &L) override;
void notifyFreeingObject(ObjectKey key) override;
public:
DebugListener() : llvm::JITEventListener(), objects() {}
llvm::Expected<llvm::DILineInfo> symbolize(uintptr_t pc);
llvm::Expected<std::string> getPrettyBacktrace(uintptr_t pc);
std::string getPrettyBacktrace(const std::vector<uintptr_t> &backtrace);
};
} // namespace codon

93
codon/compiler/engine.cpp 100644
View File

@ -0,0 +1,93 @@
#include "engine.h"
#include "codon/compiler/memory_manager.h"
#include "codon/sir/llvm/optimize.h"
namespace codon {
namespace jit {
void Engine::handleLazyCallThroughError() {
llvm::errs() << "LazyCallThrough error: Could not find function body";
exit(1);
}
llvm::Expected<llvm::orc::ThreadSafeModule>
Engine::optimizeModule(llvm::orc::ThreadSafeModule module,
const llvm::orc::MaterializationResponsibility &R) {
module.withModuleDo([](llvm::Module &module) {
ir::optimize(&module, /*debug=*/true, /*jit=*/true);
});
return std::move(module);
}
Engine::Engine(std::unique_ptr<llvm::orc::TargetProcessControl> tpc,
std::unique_ptr<llvm::orc::ExecutionSession> sess,
std::unique_ptr<llvm::orc::TPCIndirectionUtils> tpciu,
llvm::orc::JITTargetMachineBuilder jtmb, llvm::DataLayout layout)
: tpc(std::move(tpc)), sess(std::move(sess)), tpciu(std::move(tpciu)),
layout(std::move(layout)), mangle(*this->sess, this->layout),
objectLayer(*this->sess,
[]() { return std::make_unique<BoehmGCMemoryManager>(); }),
compileLayer(*this->sess, objectLayer,
std::make_unique<llvm::orc::ConcurrentIRCompiler>(std::move(jtmb))),
optimizeLayer(*this->sess, compileLayer, optimizeModule),
codLayer(*this->sess, optimizeLayer, this->tpciu->getLazyCallThroughManager(),
[this] { return this->tpciu->createIndirectStubsManager(); }),
mainJD(this->sess->createBareJITDylib("<main>")),
dbListener(std::make_unique<DebugListener>()) {
mainJD.addGenerator(
llvm::cantFail(llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
layout.getGlobalPrefix())));
objectLayer.setAutoClaimResponsibilityForObjectSymbols(true);
objectLayer.registerJITEventListener(*dbListener);
}
Engine::~Engine() {
if (auto err = sess->endSession())
sess->reportError(std::move(err));
if (auto err = tpciu->cleanup())
sess->reportError(std::move(err));
}
llvm::Expected<std::unique_ptr<Engine>> Engine::create() {
auto ssp = std::make_shared<llvm::orc::SymbolStringPool>();
auto tpc = llvm::orc::SelfTargetProcessControl::Create(ssp);
if (!tpc)
return tpc.takeError();
auto sess = std::make_unique<llvm::orc::ExecutionSession>(std::move(ssp));
auto tpciu = llvm::orc::TPCIndirectionUtils::Create(**tpc);
if (!tpciu)
return tpciu.takeError();
(*tpciu)->createLazyCallThroughManager(
*sess, llvm::pointerToJITTargetAddress(&handleLazyCallThroughError));
if (auto err = llvm::orc::setUpInProcessLCTMReentryViaTPCIU(**tpciu))
return std::move(err);
llvm::orc::JITTargetMachineBuilder jtmb((*tpc)->getTargetTriple());
auto layout = jtmb.getDefaultDataLayoutForTarget();
if (!layout)
return layout.takeError();
return std::make_unique<Engine>(std::move(*tpc), std::move(sess), std::move(*tpciu),
std::move(jtmb), std::move(*layout));
}
llvm::Error Engine::addModule(llvm::orc::ThreadSafeModule module,
llvm::orc::ResourceTrackerSP rt) {
if (!rt)
rt = mainJD.getDefaultResourceTracker();
return optimizeLayer.add(rt, std::move(module));
}
llvm::Expected<llvm::JITEvaluatedSymbol> Engine::lookup(llvm::StringRef name) {
return sess->lookup({&mainJD}, mangle(name.str()));
}
} // namespace jit
} // namespace codon

70
codon/compiler/engine.h 100644
View File

@ -0,0 +1,70 @@
#pragma once
#include <memory>
#include <vector>
#include "codon/compiler/debug_listener.h"
#include "codon/sir/llvm/llvm.h"
#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h"
#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h"
namespace codon {
namespace jit {
class Engine {
private:
std::unique_ptr<llvm::orc::TargetProcessControl> tpc;
std::unique_ptr<llvm::orc::ExecutionSession> sess;
std::unique_ptr<llvm::orc::TPCIndirectionUtils> tpciu;
llvm::DataLayout layout;
llvm::orc::MangleAndInterner mangle;
llvm::orc::RTDyldObjectLinkingLayer objectLayer;
llvm::orc::IRCompileLayer compileLayer;
llvm::orc::IRTransformLayer optimizeLayer;
llvm::orc::CompileOnDemandLayer codLayer;
llvm::orc::JITDylib &mainJD;
std::unique_ptr<DebugListener> dbListener;
static void handleLazyCallThroughError();
static llvm::Expected<llvm::orc::ThreadSafeModule>
optimizeModule(llvm::orc::ThreadSafeModule module,
const llvm::orc::MaterializationResponsibility &R);
public:
Engine(std::unique_ptr<llvm::orc::TargetProcessControl> tpc,
std::unique_ptr<llvm::orc::ExecutionSession> sess,
std::unique_ptr<llvm::orc::TPCIndirectionUtils> tpciu,
llvm::orc::JITTargetMachineBuilder jtmb, llvm::DataLayout layout);
~Engine();
static llvm::Expected<std::unique_ptr<Engine>> create();
const llvm::DataLayout &getDataLayout() const { return layout; }
llvm::orc::JITDylib &getMainJITDylib() { return mainJD; }
DebugListener *getDebugListener() const { return dbListener.get(); }
llvm::Error addModule(llvm::orc::ThreadSafeModule module,
llvm::orc::ResourceTrackerSP rt = nullptr);
llvm::Expected<llvm::JITEvaluatedSymbol> lookup(llvm::StringRef name);
};
} // namespace jit
} // namespace codon

15
codon/compiler/error.cpp 100644
View File

@ -0,0 +1,15 @@
#include "error.h"
namespace codon {
namespace error {
char ParserErrorInfo::ID = 0;
char RuntimeErrorInfo::ID = 0;
char PluginErrorInfo::ID = 0;
char IOErrorInfo::ID = 0;
} // namespace error
} // namespace codon

150
codon/compiler/error.h 100644
View File

@ -0,0 +1,150 @@
#pragma once
#include <string>
#include <vector>
#include "codon/parser/common.h"
#include "llvm/Support/Error.h"
namespace codon {
namespace error {
class Message {
private:
std::string msg;
std::string file;
int line = 0;
int col = 0;
public:
explicit Message(const std::string &msg, const std::string &file = "", int line = 0,
int col = 0)
: msg(msg), file(file), line(line), col(col) {}
std::string getMessage() const { return msg; }
std::string getFile() const { return file; }
int getLine() const { return line; }
int getColumn() const { return col; }
void log(llvm::raw_ostream &out) const {
if (!getFile().empty()) {
out << getFile();
if (getLine() != 0) {
out << ":" << getLine();
if (getColumn() != 0) {
out << ":" << getColumn();
}
}
out << ": ";
}
out << getMessage();
}
};
class ParserErrorInfo : public llvm::ErrorInfo<ParserErrorInfo> {
private:
std::vector<Message> messages;
public:
explicit ParserErrorInfo(std::vector<Message> messages)
: messages(std::move(messages)) {}
explicit ParserErrorInfo(const exc::ParserException &e) : messages() {
for (unsigned i = 0; i < e.messages.size(); i++) {
if (!e.messages[i].empty())
messages.emplace_back(e.messages[i], e.locations[i].file, e.locations[i].line,
e.locations[i].col);
}
}
auto begin() { return messages.begin(); }
auto end() { return messages.end(); }
auto begin() const { return messages.begin(); }
auto end() const { return messages.end(); }
void log(llvm::raw_ostream &out) const override {
for (auto &msg : *this) {
msg.log(out);
out << "\n";
}
}
std::error_code convertToErrorCode() const override {
return llvm::inconvertibleErrorCode();
}
static char ID;
};
class RuntimeErrorInfo : public llvm::ErrorInfo<RuntimeErrorInfo> {
private:
std::string output;
std::string type;
Message message;
std::vector<std::string> backtrace;
public:
RuntimeErrorInfo(const std::string &output, const std::string &type,
const std::string &msg, const std::string &file = "", int line = 0,
int col = 0, std::vector<std::string> backtrace = {})
: output(output), type(type), message(msg, file, line, col),
backtrace(std::move(backtrace)) {}
std::string getOutput() const { return output; }
std::string getType() const { return type; }
std::string getMessage() const { return message.getMessage(); }
std::string getFile() const { return message.getFile(); }
int getLine() const { return message.getLine(); }
int getColumn() const { return message.getColumn(); }
std::vector<std::string> getBacktrace() const { return backtrace; }
void log(llvm::raw_ostream &out) const override {
out << type << ": ";
message.log(out);
}
std::error_code convertToErrorCode() const override {
return llvm::inconvertibleErrorCode();
}
static char ID;
};
class PluginErrorInfo : public llvm::ErrorInfo<PluginErrorInfo> {
private:
std::string message;
public:
explicit PluginErrorInfo(const std::string &message) : message(message) {}
std::string getMessage() const { return message; }
void log(llvm::raw_ostream &out) const override { out << message; }
std::error_code convertToErrorCode() const override {
return llvm::inconvertibleErrorCode();
}
static char ID;
};
class IOErrorInfo : public llvm::ErrorInfo<IOErrorInfo> {
private:
std::string message;
public:
explicit IOErrorInfo(const std::string &message) : message(message) {}
std::string getMessage() const { return message; }
void log(llvm::raw_ostream &out) const override { out << message; }
std::error_code convertToErrorCode() const override {
return llvm::inconvertibleErrorCode();
}
static char ID;
};
} // namespace error
} // namespace codon

166
codon/compiler/jit.cpp 100644
View File

@ -0,0 +1,166 @@
#include "jit.h"
#include "codon/parser/peg/peg.h"
#include "codon/parser/visitors/doc/doc.h"
#include "codon/parser/visitors/format/format.h"
#include "codon/parser/visitors/simplify/simplify.h"
#include "codon/parser/visitors/translate/translate.h"
#include "codon/parser/visitors/typecheck/typecheck.h"
#include "codon/runtime/lib.h"
namespace codon {
namespace jit {
namespace {
typedef int MainFunc(int, char **);
typedef void InputFunc();
const std::string JIT_FILENAME = "<jit>";
} // namespace
JIT::JIT(const std::string &argv0, const std::string &mode)
: compiler(std::make_unique<Compiler>(argv0, /*debug=*/true)), mode(mode) {
if (auto e = Engine::create()) {
engine = std::move(e.get());
} else {
engine = {};
seqassert(false, "JIT engine creation error");
}
compiler->getLLVMVisitor()->setJIT(true);
}
llvm::Error JIT::init() {
auto *cache = compiler->getCache();
auto *module = compiler->getModule();
auto *pm = compiler->getPassManager();
auto *llvisitor = compiler->getLLVMVisitor();
auto transformed = ast::SimplifyVisitor::apply(
cache, std::make_shared<ast::SuiteStmt>(), JIT_FILENAME, {});
auto typechecked = ast::TypecheckVisitor::apply(cache, std::move(transformed));
ast::TranslateVisitor::apply(cache, std::move(typechecked));
cache->isJit = true; // we still need main(), so set isJit after it has been set
module->setSrcInfo({JIT_FILENAME, 0, 0, 0});
pm->run(module);
module->accept(*llvisitor);
auto pair = llvisitor->takeModule();
if (auto err = engine->addModule({std::move(pair.first), std::move(pair.second)}))
return err;
auto func = engine->lookup("main");
if (auto err = func.takeError())
return err;
auto *main = (MainFunc *)func->getAddress();
(*main)(0, nullptr);
return llvm::Error::success();
}
llvm::Expected<std::string> JIT::run(const ir::Func *input,
const std::vector<ir::Var *> &newGlobals) {
auto *module = compiler->getModule();
auto *pm = compiler->getPassManager();
auto *llvisitor = compiler->getLLVMVisitor();
pm->run(module);
const std::string name = ir::LLVMVisitor::getNameForFunction(input);
llvisitor->registerGlobal(input);
for (auto *var : newGlobals) {
llvisitor->registerGlobal(var);
}
for (auto *var : newGlobals) {
if (auto *func = ir::cast<ir::Func>(var))
func->accept(*llvisitor);
}
input->accept(*llvisitor);
auto pair = llvisitor->takeModule();
if (auto err = engine->addModule({std::move(pair.first), std::move(pair.second)}))
return std::move(err);
auto func = engine->lookup(name);
if (auto err = func.takeError())
return std::move(err);
auto *repl = (InputFunc *)func->getAddress();
try {
(*repl)();
} catch (const JITError &e) {
std::vector<std::string> backtrace;
for (auto pc : e.getBacktrace()) {
auto line = engine->getDebugListener()->getPrettyBacktrace(pc);
if (line && !line->empty())
backtrace.push_back(*line);
}
return llvm::make_error<error::RuntimeErrorInfo>(e.getOutput(), e.getType(),
e.what(), e.getFile(), e.getLine(),
e.getCol(), backtrace);
}
return getCapturedOutput();
}
llvm::Expected<std::string> JIT::exec(const std::string &code) {
auto *cache = compiler->getCache();
ast::StmtPtr node = ast::parseCode(cache, JIT_FILENAME, code, /*startLine=*/0);
auto sctx = cache->imports[MAIN_IMPORT].ctx;
auto preamble = std::make_shared<ast::SimplifyVisitor::Preamble>();
try {
auto *e = node->getSuite()
? const_cast<ast::SuiteStmt *>(node->getSuite())->lastInBlock()
: &node;
if (e)
if (auto ex = (*e)->getExpr()) {
*e = std::make_shared<ast::PrintStmt>(
std::vector<ast::ExprPtr>{std::make_shared<ast::CallExpr>(
std::make_shared<ast::IdExpr>("_jit_display"), ex->expr,
std::make_shared<ast::StringExpr>(mode))},
false);
}
auto s = ast::SimplifyVisitor(sctx, preamble).transform(node);
auto simplified = std::make_shared<ast::SuiteStmt>();
for (auto &s : preamble->globals)
simplified->stmts.push_back(s);
for (auto &s : preamble->functions)
simplified->stmts.push_back(s);
simplified->stmts.push_back(s);
// TODO: unroll on errors...
auto *cache = compiler->getCache();
auto typechecked = ast::TypecheckVisitor::apply(cache, simplified);
std::vector<std::string> globalNames;
for (auto &g : cache->globals) {
if (!g.second)
globalNames.push_back(g.first);
}
// add newly realized functions
std::vector<ast::StmtPtr> v;
std::vector<ir::Func **> frs;
v.push_back(typechecked);
for (auto &p : cache->pendingRealizations) {
v.push_back(cache->functions[p.first].ast);
frs.push_back(&cache->functions[p.first].realizations[p.second]->ir);
}
auto func =
ast::TranslateVisitor::apply(cache, std::make_shared<ast::SuiteStmt>(v, false));
cache->jitCell++;
std::vector<ir::Var *> globalVars;
for (auto &g : globalNames) {
seqassert(cache->globals[g], "JIT global {} not set", g);
globalVars.push_back(cache->globals[g]);
}
for (auto &i : frs) {
seqassert(*i, "JIT fn not set");
globalVars.push_back(*i);
}
return run(func, globalVars);
} catch (const exc::ParserException &e) {
return llvm::make_error<error::ParserErrorInfo>(e);
}
}
} // namespace jit
} // namespace codon

37
codon/compiler/jit.h 100644
View File

@ -0,0 +1,37 @@
#pragma once
#include <memory>
#include <string>
#include <vector>
#include "codon/compiler/compiler.h"
#include "codon/compiler/engine.h"
#include "codon/compiler/error.h"
#include "codon/parser/cache.h"
#include "codon/sir/llvm/llvisitor.h"
#include "codon/sir/transform/manager.h"
#include "codon/sir/var.h"
namespace codon {
namespace jit {
class JIT {
private:
std::unique_ptr<Compiler> compiler;
std::unique_ptr<Engine> engine;
std::string mode;
public:
explicit JIT(const std::string &argv0, const std::string &mode = "");
Compiler *getCompiler() const { return compiler.get(); }
Engine *getEngine() const { return engine.get(); }
llvm::Error init();
llvm::Expected<std::string> run(const ir::Func *input,
const std::vector<ir::Var *> &newGlobals = {});
llvm::Expected<std::string> exec(const std::string &code);
};
} // namespace jit
} // namespace codon

2
codon/sir/llvm/memory_manager.cpp → codon/compiler/memory_manager.cpp
View File

@ -3,7 +3,6 @@
#include "codon/runtime/lib.h"
namespace codon {
namespace ir {
BoehmGCMemoryManager::BoehmGCMemoryManager() : SectionMemoryManager(), roots() {}
@ -26,5 +25,4 @@ BoehmGCMemoryManager::~BoehmGCMemoryManager() {
}
}
} // namespace ir
} // namespace codon

2
codon/sir/llvm/memory_manager.h → codon/compiler/memory_manager.h
View File

@ -6,7 +6,6 @@
#include "codon/sir/llvm/llvm.h"
namespace codon {
namespace ir {
/// Simple extension of LLVM's SectionMemoryManager which catches data section
/// allocations and registers them with the GC. This allows the GC to know not
@ -23,5 +22,4 @@ public:
~BoehmGCMemoryManager() override;
};
} // namespace ir
} // namespace codon

38
codon/dsl/plugins.cpp
View File

@ -10,10 +10,8 @@
namespace codon {
namespace {
bool error(const std::string &msg, std::string *errMsg) {
if (!msg.empty() && errMsg)
*errMsg = msg;
return false;
llvm::Expected<Plugin *> pluginError(const std::string &msg) {
return llvm::make_error<error::PluginErrorInfo>(msg);
}
typedef std::unique_ptr<DSL> LoadFunc();
@ -21,7 +19,7 @@ typedef std::unique_ptr<DSL> LoadFunc();
namespace fs = std::filesystem;
bool PluginManager::load(const std::string &path, std::string *errMsg) {
llvm::Expected<Plugin *> PluginManager::load(const std::string &path) {
#if __APPLE__
const std::string libExt = "dylib";
#else
@ -40,9 +38,8 @@ bool PluginManager::load(const std::string &path, std::string *errMsg) {
try {
tml = toml::parse_file(tomlPath.string());
} catch (const toml::parse_error &e) {
return error(
fmt::format("[toml::parse_file(\"{}\")] {}", tomlPath.string(), e.what()),
errMsg);
return pluginError(
fmt::format("[toml::parse_file(\"{}\")] {}", tomlPath.string(), e.what()));
}
auto about = tml["about"];
auto library = tml["library"];
@ -70,31 +67,26 @@ bool PluginManager::load(const std::string &path, std::string *errMsg) {
semver::version(CODON_VERSION_MAJOR, CODON_VERSION_MINOR, CODON_VERSION_PATCH),
info.supported);
} catch (const std::invalid_argument &e) {
return error(fmt::format("[semver::range::satisfies(..., \"{}\")] {}",
info.supported, e.what()),
errMsg);
return pluginError(fmt::format("[semver::range::satisfies(..., \"{}\")] {}",
info.supported, e.what()));
}
if (!versionOk)
return error(fmt::format("unsupported version {} (supported: {})", CODON_VERSION,
info.supported),
errMsg);
return pluginError(fmt::format("unsupported version {} (supported: {})",
CODON_VERSION, info.supported));
if (!dylibPath.empty()) {
std::string libLoadErrorMsg;
auto handle = llvm::sys::DynamicLibrary::getPermanentLibrary(dylibPath.c_str(),
&libLoadErrorMsg);
if (!handle.isValid())
return error(
fmt::format(
"[llvm::sys::DynamicLibrary::getPermanentLibrary(\"{}\", ...)] {}",
dylibPath, libLoadErrorMsg),
errMsg);
return pluginError(fmt::format(
"[llvm::sys::DynamicLibrary::getPermanentLibrary(\"{}\", ...)] {}", dylibPath,
libLoadErrorMsg));
auto *entry = (LoadFunc *)handle.getAddressOfSymbol("load");
if (!entry)
return error(
fmt::format("could not find 'load' in plugin shared library: {}", dylibPath),
errMsg);
return pluginError(
fmt::format("could not find 'load' in plugin shared library: {}", dylibPath));
auto dsl = (*entry)();
plugins.push_back(std::make_unique<Plugin>(std::move(dsl), info, handle));
@ -102,7 +94,7 @@ bool PluginManager::load(const std::string &path, std::string *errMsg) {
plugins.push_back(std::make_unique<Plugin>(std::make_unique<DSL>(), info,
llvm::sys::DynamicLibrary()));
}
return true;
return plugins.back().get();
}
} // namespace codon

6
codon/dsl/plugins.h
View File

@ -5,6 +5,7 @@
#include <string>
#include <vector>
#include "codon/compiler/error.h"
#include "codon/dsl/dsl.h"
#include "codon/sir/util/iterators.h"
#include "llvm/Support/DynamicLibrary.h"
@ -45,9 +46,8 @@ public:
/// Loads the plugin at the given load path.
/// @param path path to plugin directory containing "plugin.toml" file
/// @param errMsg where to store potential error messages, if non-null
/// @return true if the plugin was loaded successfully, false otherwise
bool load(const std::string &path, std::string *errMsg = nullptr);
/// @return plugin pointer if successful, plugin error otherwise
llvm::Expected<Plugin *> load(const std::string &path);
};
} // namespace codon

12
codon/parser/ast/stmt.cpp
View File

@ -52,6 +52,16 @@ void SuiteStmt::flatten(StmtPtr s, std::vector<StmtPtr> &stmts) {
stmts.push_back(ss);
}
}
StmtPtr *SuiteStmt::lastInBlock() {
if (stmts.empty())
return nullptr;
if (auto s = const_cast<SuiteStmt *>(stmts.back()->getSuite())) {
auto l = s->lastInBlock();
if (l)
return l;
}
return &(stmts.back());
}
std::string BreakStmt::toString(int) const { return "(break)"; }
ACCEPT_IMPL(BreakStmt, ASTVisitor);
@ -362,7 +372,7 @@ ACCEPT_IMPL(YieldFromStmt, ASTVisitor);
WithStmt::WithStmt(std::vector<ExprPtr> items, std::vector<std::string> vars,
StmtPtr suite)
: Stmt(), items(std::move(items)), vars(std::move(vars)), suite(std::move(suite)) {
assert(items.size() == vars.size());
seqassert(items.size() == vars.size(), "vector size mismatch");
}
WithStmt::WithStmt(std::vector<std::pair<ExprPtr, ExprPtr>> itemVarPairs, StmtPtr suite)
: Stmt(), suite(std::move(suite)) {

1
codon/parser/ast/stmt.h
View File

@ -92,6 +92,7 @@ struct SuiteStmt : public Stmt {
const Stmt *firstInBlock() const override {
return stmts.empty() ? nullptr : stmts[0]->firstInBlock();
}
StmtPtr *lastInBlock();
/// Flatten all nested SuiteStmt objects that do not own a block in the statement
/// vector. This is shallow flattening.

4
codon/parser/ast/types.cpp
View File

@ -18,7 +18,7 @@ void Type::Unification::undo() {
linked[i]->type = nullptr;
}
for (int i = int(leveled.size()) - 1; i >= 0; i--) {
assert(leveled[i].first->kind == LinkType::Unbound);
seqassert(leveled[i].first->kind == LinkType::Unbound, "not unbound");
leveled[i].first->level = leveled[i].second;
}
for (auto &t : traits)
@ -687,7 +687,7 @@ std::string StaticType::realizedName() const {
deps.push_back(e.type->realizedName());
if (!expr->staticValue.evaluated) // If not already evaluated, evaluate!
const_cast<StaticType *>(this)->expr->staticValue = evaluate();
assert(expr->staticValue.evaluated);
seqassert(expr->staticValue.evaluated, "static value not evaluated");
return expr->staticValue.toString();
}
StaticValue StaticType::evaluate() const {

3
codon/parser/cache.cpp
View File

@ -14,7 +14,8 @@ namespace ast {
Cache::Cache(std::string argv0)
: generatedSrcInfoCount(0), unboundCount(0), varCount(0), age(0), testFlags(0),
argv0(move(argv0)), module(nullptr), typeCtx(nullptr), codegenCtx(nullptr) {}
argv0(move(argv0)), module(nullptr), typeCtx(nullptr), codegenCtx(nullptr),
isJit(false), jitCell(0) {}
std::string Cache::getTemporaryVar(const std::string &prefix, char sigil) {
return fmt::format("{}{}_{}", sigil ? fmt::format("{}_", sigil) : "", prefix,

14
codon/parser/cache.h
View File

@ -88,16 +88,16 @@ struct Cache : public std::enable_shared_from_this<Cache> {
std::string argv0;
/// Absolute path of the entry-point module (if available).
std::string module0;
/// LLVM module.
codon::ir::Module *module = nullptr;
/// IR module.
ir::Module *module = nullptr;
/// Table of imported files that maps an absolute filename to a Import structure.
/// By convention, the key of Seq standard library is "".
/// By convention, the key of the Codon's standard library is "".
std::unordered_map<std::string, Import> imports;
/// Set of unique (canonical) global identifiers for marking such variables as global
/// in code-generation step.
std::set<std::string> globals;
/// in code-generation step and in JIT.
std::map<std::string, ir::Var *> globals;
/// Stores class data for each class (type) in the source code.
struct Class {
@ -195,6 +195,10 @@ struct Cache : public std::enable_shared_from_this<Cache> {
/// Plugin-added import paths
std::vector<std::string> pluginImportPaths;
/// Set if the Codon is running in JIT mode.
bool isJit;
int jitCell;
public:
explicit Cache(std::string argv0 = "");

130
codon/parser/common.cpp
View File

@ -1,8 +1,8 @@
#include "common.h"
#include <libgen.h>
#include <filesystem>
#include <string>
#include <sys/stat.h>
#include <vector>
#include "codon/parser/common.h"
#include "codon/util/fmt/format.h"
@ -195,79 +195,77 @@ std::string executable_path(const char *argv0) {
std::string executable_path(const char *argv0) { return std::string(argv0); }
#endif
namespace fs = std::filesystem;
namespace {
void addPath(std::vector<fs::path> &paths, const fs::path &path) {
if (fs::exists(path))
paths.push_back(fs::canonical(path));
}
std::vector<fs::path> getStdLibPaths(const std::string &argv0,
const std::vector<std::string> &plugins) {
std::vector<fs::path> paths;
if (auto c = getenv("CODON_PATH")) {
addPath(paths, fs::path(std::string(c)));
}
if (!argv0.empty()) {
auto base = fs::path(executable_path(argv0.c_str()));
for (auto loci : {"../lib/codon/stdlib", "../stdlib", "stdlib"}) {
addPath(paths, base.parent_path() / loci);
}
}
for (auto &path : plugins) {
addPath(paths, fs::path(path));
}
return paths;
}
ImportFile getRoot(const std::string argv0, const std::vector<std::string> &plugins,
const std::string &module0Root, const std::string &s) {
bool isStdLib = false;
std::string root;
for (auto &p : getStdLibPaths(argv0, plugins))
if (startswith(s, p)) {
root = p;
isStdLib = true;
break;
}
if (!isStdLib && startswith(s, module0Root))
root = module0Root;
const std::string ext = ".codon";
seqassert(startswith(s, root) && endswith(s, ext), "bad path substitution: {}, {}", s,
root);
auto module = s.substr(root.size() + 1, s.size() - root.size() - ext.size() - 1);
std::replace(module.begin(), module.end(), '/', '.');
return ImportFile{(!isStdLib && root == module0Root) ? ImportFile::PACKAGE
: ImportFile::STDLIB,
s, module};
}
} // namespace
std::shared_ptr<ImportFile> getImportFile(const std::string &argv0,
const std::string &what,
const std::string &relativeTo,
bool forceStdlib, const std::string &module0,
const std::vector<std::string> &plugins) {
using fmt::format;
auto getStdLibPaths = [](const std::string &argv0,
const std::vector<std::string> &plugins) {
std::vector<std::string> paths;
char abs[PATH_MAX + 1];
if (auto c = getenv("CODON_PATH")) {
if (realpath(c, abs))
paths.push_back(abs);
std::vector<fs::path> paths;
if (what != "<jit>") {
auto parentRelativeTo = fs::path(relativeTo).parent_path();
if (!forceStdlib) {
addPath(paths, (parentRelativeTo / what).replace_extension("codon"));
addPath(paths, parentRelativeTo / what / "__init__.codon");
}
if (!argv0.empty()) {
for (auto loci : {"../lib/codon/stdlib", "../stdlib", "stdlib"}) {
strncpy(abs, executable_path(argv0.c_str()).c_str(), PATH_MAX);
if (realpath(format("{}/{}", dirname(abs), loci).c_str(), abs))
paths.push_back(abs);
}
}
for (auto &path : plugins) {
if (realpath(path.c_str(), abs))
paths.push_back(abs);
}
return paths;
};
char abs[PATH_MAX + 1];
strncpy(abs, module0.c_str(), PATH_MAX);
auto module0Root = std::string(dirname(abs));
auto getRoot = [&](const std::string &s) {
bool isStdLib = false;
std::string root;
for (auto &p : getStdLibPaths(argv0, plugins))
if (startswith(s, p)) {
root = p;
isStdLib = true;
break;
}
if (!isStdLib && startswith(s, module0Root))
root = module0Root;
const std::string ext = ".codon";
seqassert(startswith(s, root) && endswith(s, ext), "bad path substitution: {}, {}",
s, root);
auto module = s.substr(root.size() + 1, s.size() - root.size() - ext.size() - 1);
std::replace(module.begin(), module.end(), '/', '.');
return ImportFile{(!isStdLib && root == module0Root) ? ImportFile::PACKAGE
: ImportFile::STDLIB,
s, module};
};
std::vector<std::string> paths;
if (!forceStdlib) {
realpath(relativeTo.c_str(), abs);
auto parent = dirname(abs);
paths.push_back(format("{}/{}.codon", parent, what));
paths.push_back(format("{}/{}/__init__.codon", parent, what));
}
for (auto &p : getStdLibPaths(argv0, plugins)) {
paths.push_back(format("{}/{}.codon", p, what));
paths.push_back(format("{}/{}/__init__.codon", p, what));
addPath(paths, (p / what).replace_extension("codon"));
addPath(paths, p / what / "__init__.codon");
}
for (auto &p : paths) {
if (!realpath(p.c_str(), abs))
continue;
auto path = std::string(abs);
struct stat buffer;
if (!stat(path.c_str(), &buffer))
return std::make_shared<ImportFile>(getRoot(path));
}
return nullptr;
auto module0Root = fs::path(module0).parent_path().string();
return paths.empty() ? nullptr
: std::make_shared<ImportFile>(
getRoot(argv0, plugins, module0Root, paths[0].string()));
}
} // namespace ast

2
codon/parser/common.h
View File

@ -16,6 +16,8 @@
#include "codon/util/fmt/format.h"
#include "codon/util/fmt/ostream.h"
#define CAST(s, T) dynamic_cast<T *>(s.get())
namespace codon {
namespace exc {

166
codon/parser/parser.cpp
View File

@ -1,166 +0,0 @@
#include "parser.h"
#include <chrono>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include "codon/parser/cache.h"
#include "codon/parser/peg/peg.h"
#include "codon/parser/visitors/doc/doc.h"
#include "codon/parser/visitors/format/format.h"
#include "codon/parser/visitors/simplify/simplify.h"
#include "codon/parser/visitors/translate/translate.h"
#include "codon/parser/visitors/typecheck/typecheck.h"
#include "codon/sir/sir.h"
#include "codon/sir/util/format.h"
#include "codon/util/fmt/format.h"
int _ocaml_time = 0;
int _ll_time = 0;
int _level = 0;
int _dbg_level = 0;
bool _isTest = false;
namespace codon {
ir::Module *parse(const std::string &argv0, const std::string &file,
const std::string &code, bool isCode, int isTest, int startLine,
const std::unordered_map<std::string, std::string> &defines,
PluginManager *plm) {
try {
auto d = getenv("CODON_DEBUG");
if (d) {
auto s = std::string(d);
_dbg_level |= s.find('t') != std::string::npos ? (1 << 0) : 0; // time
_dbg_level |= s.find('r') != std::string::npos ? (1 << 2) : 0; // realize
_dbg_level |= s.find('T') != std::string::npos ? (1 << 4) : 0; // type-check
_dbg_level |= s.find('L') != std::string::npos ? (1 << 5) : 0; // lexer
_dbg_level |= s.find('i') != std::string::npos ? (1 << 6) : 0; // IR
_dbg_level |=
s.find('l') != std::string::npos ? (1 << 7) : 0; // User-level debugging
}
char abs[PATH_MAX + 1] = {'-', 0};
if (file != "-")
realpath(file.c_str(), abs);
auto cache = std::make_shared<ast::Cache>(argv0);
if (plm) {
for (auto *plugin : *plm) {
if (!plugin->info.stdlibPath.empty())
cache->pluginImportPaths.push_back(plugin->info.stdlibPath);
for (auto &kw : plugin->dsl->getExprKeywords()) {
cache->customExprStmts[kw.keyword] = kw.callback;
}
for (auto &kw : plugin->dsl->getBlockKeywords()) {
cache->customBlockStmts[kw.keyword] = {kw.hasExpr, kw.callback};
}
}
}
ast::StmtPtr codeStmt = isCode ? ast::parseCode(cache, abs, code, startLine)
: ast::parseFile(cache, abs);
if (_dbg_level) {
auto fo = fopen("_dump.sexp", "w");
fmt::print(fo, "{}\n", codeStmt->toString(0));
fclose(fo);
}
using namespace std::chrono;
cache->module0 = file;
if (isTest)
cache->testFlags = isTest;
auto t = high_resolution_clock::now();
auto transformed =
ast::SimplifyVisitor::apply(cache, move(codeStmt), abs, defines, (isTest > 1));
if (!isTest) {
LOG_TIME("[T] ocaml = {:.1f}", _ocaml_time / 1000.0);
LOG_TIME("[T] simplify = {:.1f}",
(duration_cast<milliseconds>(high_resolution_clock::now() - t).count() -
_ocaml_time) /
1000.0);
if (_dbg_level) {
auto fo = fopen("_dump_simplify.sexp", "w");
fmt::print(fo, "{}\n", transformed->toString(0));
fclose(fo);
fo = fopen("_dump_simplify.codon", "w");
fmt::print(fo, "{}", ast::FormatVisitor::apply(transformed, cache));
fclose(fo);
}
}
t = high_resolution_clock::now();
auto typechecked = ast::TypecheckVisitor::apply(cache, move(transformed));
if (!isTest) {
LOG_TIME("[T] typecheck = {:.1f}",
duration_cast<milliseconds>(high_resolution_clock::now() - t).count() /
1000.0);
if (_dbg_level) {
auto fo = fopen("_dump_typecheck.codon", "w");
fmt::print(fo, "{}", ast::FormatVisitor::apply(typechecked, cache));
fclose(fo);
fo = fopen("_dump_typecheck.sexp", "w");
fmt::print(fo, "{}\n", typechecked->toString(0));
for (auto &f : cache->functions)
for (auto &r : f.second.realizations)
fmt::print(fo, "{}\n", r.second->ast->toString(0));
fclose(fo);
}
}
t = high_resolution_clock::now();
auto *module = ast::TranslateVisitor::apply(cache, move(typechecked));
module->setSrcInfo({abs, 0, 0, 0});
if (!isTest)
LOG_TIME("[T] translate = {:.1f}",
duration_cast<milliseconds>(high_resolution_clock::now() - t).count() /
1000.0);
if (_dbg_level) {
auto out = codon::ir::util::format(module);
std::ofstream os("_dump_sir.lisp");
os << out;
os.close();
os.close();
}
_isTest = isTest;
return module;
} catch (exc::ParserException &e) {
for (int i = 0; i < e.messages.size(); i++)
if (!e.messages[i].empty()) {
if (isTest) {
_level = 0;
LOG("{}", e.messages[i]);
} else {
compilationError(e.messages[i], e.locations[i].file, e.locations[i].line,
e.locations[i].col, /*terminate=*/false);
}
}
return nullptr;
}
}
void generateDocstr(const std::string &argv0) {
std::vector<std::string> files;
std::string s;
while (std::getline(std::cin, s))
files.push_back(s);
try {
auto j = ast::DocVisitor::apply(argv0, files);
fmt::print("{}\n", j->toString());
} catch (exc::ParserException &e) {
for (int i = 0; i < e.messages.size(); i++)
if (!e.messages[i].empty()) {
compilationError(e.messages[i], e.locations[i].file, e.locations[i].line,
e.locations[i].col, /*terminate=*/false);
}
}
}
} // namespace codon

22
codon/parser/parser.h
View File

@ -1,22 +0,0 @@
#pragma once
#include <string>
#include <unordered_map>
#include <vector>
#include "codon/dsl/plugins.h"
#include "codon/sir/sir.h"
#include "codon/util/common.h"
namespace codon {
codon::ir::Module *parse(const std::string &argv0, const std::string &file,
const std::string &code = "", bool isCode = false,
int isTest = 0, int startLine = 0,
const std::unordered_map<std::string, std::string> &defines =
std::unordered_map<std::string, std::string>{},
PluginManager *plm = nullptr);
void generateDocstr(const std::string &argv0);
} // namespace codon

59
codon/parser/peg/peg.cpp
View File

@ -14,9 +14,6 @@
#include "codon/parser/visitors/format/format.h"
#include "codon/util/cpp-peglib/peglib.h"
extern int _ocaml_time;
using namespace std;
namespace codon {
namespace ast {
@ -27,9 +24,12 @@ std::shared_ptr<peg::Grammar> initParser() {
auto g = std::make_shared<peg::Grammar>();
init_codon_rules(*g);
init_codon_actions(*g);
~(*g)["NLP"] <= peg::usr([](const char *s, size_t n, peg::SemanticValues &, any &dt) {
return any_cast<ParseContext &>(dt).parens ? 0 : (n >= 1 && s[0] == '\\' ? 1 : -1);
});
~(*g)["NLP"] <=
peg::usr([](const char *s, size_t n, peg::SemanticValues &, std::any &dt) {
return std::any_cast<ParseContext &>(dt).parens
? 0
: (n >= 1 && s[0] == '\\' ? 1 : -1);
});
for (auto &x : *g) {
auto v = peg::LinkReferences(*g, x.second.params);
x.second.accept(v);
@ -38,33 +38,32 @@ std::shared_ptr<peg::Grammar> initParser() {
for (auto &rule : std::vector<std::string>{
"arguments", "slices", "genexp", "parentheses", "star_parens", "generics",
"with_parens_item", "params", "from_as_parens", "from_params"}) {
(*g)[rule].enter = [](const char *, size_t, any &dt) {
any_cast<ParseContext &>(dt).parens++;
(*g)[rule].enter = [](const char *, size_t, std::any &dt) {
std::any_cast<ParseContext &>(dt).parens++;
};
(*g)[rule.c_str()].leave = [](const char *, size_t, size_t, any &, any &dt) {
any_cast<ParseContext &>(dt).parens--;
(*g)[rule.c_str()].leave = [](const char *, size_t, size_t, std::any &,
std::any &dt) {
std::any_cast<ParseContext &>(dt).parens--;
};
}
return g;
}
template <typename T>
T parseCode(const std::shared_ptr<Cache> &cache, const std::string &file,
std::string code, int line_offset, int col_offset,
const std::string &rule) {
using namespace std::chrono;
auto t = high_resolution_clock::now();
T parseCode(Cache *cache, const std::string &file, std::string code, int line_offset,
int col_offset, const std::string &rule) {
TIME("peg");
// Initialize
if (!grammar)
grammar = initParser();
std::vector<tuple<size_t, size_t, std::string>> errors;
std::vector<std::tuple<size_t, size_t, std::string>> errors;
auto log = [&](size_t line, size_t col, const std::string &msg) {
errors.push_back({line, col, msg});
};
T result = nullptr;
auto ctx = make_any<ParseContext>(cache, 0, line_offset, col_offset);
auto ctx = std::make_any<ParseContext>(cache, 0, line_offset, col_offset);
auto r = (*grammar)[rule].parse_and_get_value(code.c_str(), code.size(), ctx, result,
file.c_str(), log);
auto ret = r.ret && r.len == code.size();
@ -73,35 +72,34 @@ T parseCode(const std::shared_ptr<Cache> &cache, const std::string &file,
exc::ParserException ex;
if (!errors.empty()) {
for (auto &e : errors)
ex.track(fmt::format("{}", get<2>(e)), SrcInfo(file, get<0>(e), get<1>(e), 0));
ex.track(fmt::format("{}", std::get<2>(e)),
SrcInfo(file, std::get<0>(e), std::get<1>(e), 0));
throw ex;
return nullptr;
}
_ocaml_time += duration_cast<milliseconds>(high_resolution_clock::now() - t).count();
return result;
}
StmtPtr parseCode(const std::shared_ptr<Cache> &cache, const std::string &file,
const std::string &code, int line_offset) {
StmtPtr parseCode(Cache *cache, const std::string &file, const std::string &code,
int line_offset) {
return parseCode<StmtPtr>(cache, file, code + "\n", line_offset, 0, "program");
}
ExprPtr parseExpr(const std::shared_ptr<Cache> &cache, const std::string &code,
const codon::SrcInfo &offset) {
ExprPtr parseExpr(Cache *cache, const std::string &code, const codon::SrcInfo &offset) {
return parseCode<ExprPtr>(cache, offset.file, code, offset.line, offset.col,
"fstring");
}
StmtPtr parseFile(const std::shared_ptr<Cache> &cache, const std::string &file) {
StmtPtr parseFile(Cache *cache, const std::string &file) {
std::vector<std::string> lines;
std::string code;
if (file == "-") {
for (std::string line; getline(cin, line);) {
for (std::string line; getline(std::cin, line);) {
lines.push_back(line);
code += line + "\n";
}
} else {
ifstream fin(file);
std::ifstream fin(file);
if (!fin)
error(fmt::format("cannot open {}", file).c_str());
for (std::string line; getline(fin, line);) {
@ -131,18 +129,17 @@ std::shared_ptr<peg::Grammar> initOpenMPParser() {
return g;
}
std::vector<CallExpr::Arg> parseOpenMP(const std::shared_ptr<Cache> &cache,
const std::string &code,
std::vector<CallExpr::Arg> parseOpenMP(Cache *cache, const std::string &code,
const codon::SrcInfo &loc) {
if (!ompGrammar)
ompGrammar = initOpenMPParser();
std::vector<tuple<size_t, size_t, std::string>> errors;
std::vector<std::tuple<size_t, size_t, std::string>> errors;
auto log = [&](size_t line, size_t col, const std::string &msg) {
errors.push_back({line, col, msg});
};
std::vector<CallExpr::Arg> result;
auto ctx = make_any<ParseContext>(cache, 0, 0, 0);
auto ctx = std::make_any<ParseContext>(cache, 0, 0, 0);
auto r = (*ompGrammar)["pragma"].parse_and_get_value(code.c_str(), code.size(), ctx,
result, "", log);
auto ret = r.ret && r.len == code.size();
@ -150,7 +147,7 @@ std::vector<CallExpr::Arg> parseOpenMP(const std::shared_ptr<Cache> &cache,
r.error_info.output_log(log, code.c_str(), code.size());
exc::ParserException ex;
if (!errors.empty()) {
ex.track(fmt::format("openmp {}", get<2>(errors[0])), loc);
ex.track(fmt::format("openmp {}", std::get<2>(errors[0])), loc);
throw ex;
}
return result;

12
codon/parser/peg/peg.h
View File

@ -12,17 +12,15 @@ namespace codon {
namespace ast {
/// Parse a Seq code block with the appropriate file and position offsets.
StmtPtr parseCode(const std::shared_ptr<Cache> &cache, const std::string &file,
const std::string &code, int line_offset = 0);
StmtPtr parseCode(Cache *cache, const std::string &file, const std::string &code,
int line_offset = 0);
/// Parse a Seq code expression.
ExprPtr parseExpr(const std::shared_ptr<Cache> &cache, const std::string &code,
const codon::SrcInfo &offset);
ExprPtr parseExpr(Cache *cache, const std::string &code, const codon::SrcInfo &offset);
/// Parse a Seq file.
StmtPtr parseFile(const std::shared_ptr<Cache> &cache, const std::string &file);
StmtPtr parseFile(Cache *cache, const std::string &file);
/// Parse a OpenMP clause.
std::vector<CallExpr::Arg> parseOpenMP(const std::shared_ptr<Cache> &cache,
const std::string &code,
std::vector<CallExpr::Arg> parseOpenMP(Cache *cache, const std::string &code,
const codon::SrcInfo &loc);
} // namespace ast

9
codon/parser/peg/rules.h
View File

@ -17,14 +17,13 @@ namespace codon {
namespace ast {
struct ParseContext {
std::shared_ptr<Cache> cache;
Cache *cache;
std::stack<int> indent;
int parens;
int line_offset, col_offset;
ParseContext(std::shared_ptr<Cache> cache, int parens = 0, int line_offset = 0,
int col_offset = 0)
: cache(move(cache)), parens(parens), line_offset(line_offset),
col_offset(col_offset) {}
ParseContext(Cache *cache, int parens = 0, int line_offset = 0, int col_offset = 0)
: cache(cache), parens(parens), line_offset(line_offset), col_offset(col_offset) {
}
bool hasCustomStmtKeyword(const std::string &kwd, bool hasExpr) const {
auto i = cache->customBlockStmts.find(kwd);

12
codon/parser/visitors/doc/doc.cpp
View File

@ -1,5 +1,6 @@
#include "doc.h"
#include <filesystem>
#include <memory>
#include <string>
#include <tuple>
@ -89,7 +90,8 @@ std::shared_ptr<json> DocVisitor::apply(const std::string &argv0,
const std::vector<std::string> &files) {
auto shared = std::make_shared<DocShared>();
shared->argv0 = argv0;
shared->cache = std::make_shared<ast::Cache>(argv0);
auto cache = std::make_unique<ast::Cache>(argv0);
shared->cache = cache.get();
auto stdlib = getImportFile(argv0, "internal", "", true, "");
auto ast = ast::parseFile(shared->cache, stdlib->path);
@ -113,15 +115,15 @@ std::shared_ptr<json> DocVisitor::apply(const std::string &argv0,
DocVisitor(shared->modules[""]).transformModule(std::move(ast));
auto ctx = std::make_shared<DocContext>(shared);
char abs[PATH_MAX];
for (auto &f : files) {
realpath(f.c_str(), abs);
ctx->setFilename(abs);
ast = ast::parseFile(shared->cache, abs);
auto path = std::filesystem::canonical(std::filesystem::path(f)).string();
ctx->setFilename(path);
ast = ast::parseFile(shared->cache, path);
// LOG("parsing {}", f);
DocVisitor(ctx).transformModule(std::move(ast));
}
shared->cache = nullptr;
return shared->j;
}

2
codon/parser/visitors/doc/doc.h
View File

@ -41,7 +41,7 @@ struct DocShared {
std::shared_ptr<json> j;
std::unordered_map<std::string, std::shared_ptr<DocContext>> modules;
std::string argv0;
std::shared_ptr<Cache> cache;
Cache *cache;
std::unordered_map<int, std::vector<std::string>> generics;
DocShared() : itemID(1) {}
};

2
codon/parser/visitors/format/format.cpp
View File

@ -9,7 +9,7 @@ using fmt::format;
namespace codon {
namespace ast {
FormatVisitor::FormatVisitor(bool html, std::shared_ptr<Cache> cache)
FormatVisitor::FormatVisitor(bool html, Cache *cache)
: renderType(false), renderHTML(html), indent(0), cache(cache) {
if (renderHTML) {
header = "<html><head><link rel=stylesheet href=code.css/></head>\n<body>";

8
codon/parser/visitors/format/format.h
View File

@ -25,7 +25,7 @@ class FormatVisitor : public CallbackASTVisitor<std::string, std::string> {
std::string commentStart, commentEnd;
std::string keywordStart, keywordEnd;
std::shared_ptr<Cache> cache;
Cache *cache;
private:
template <typename T, typename... Ts> std::string renderExpr(T &&t, Ts &&...args) {
@ -44,15 +44,15 @@ private:
std::string keyword(const std::string &s) const;
public:
FormatVisitor(bool html, std::shared_ptr<Cache> cache = nullptr);
FormatVisitor(bool html, Cache *cache = nullptr);
std::string transform(const ExprPtr &e) override;
std::string transform(const Expr *expr);
std::string transform(const StmtPtr &stmt) override;
std::string transform(Stmt *stmt, int indent);
template <typename T>
static std::string apply(const T &stmt, std::shared_ptr<Cache> cache = nullptr,
bool html = false, bool init = false) {
static std::string apply(const T &stmt, Cache *cache = nullptr, bool html = false,
bool init = false) {
auto t = FormatVisitor(html, cache);
return fmt::format("{}{}{}", t.header, t.transform(stmt), t.footer);
}

13
codon/parser/visitors/simplify/simplify.cpp
View File

@ -17,14 +17,13 @@ namespace ast {
using namespace types;
StmtPtr SimplifyVisitor::apply(
std::shared_ptr<Cache> cache, const StmtPtr &node, const std::string &file,
const std::unordered_map<std::string, std::string> &defines, bool barebones) {
StmtPtr
SimplifyVisitor::apply(Cache *cache, const StmtPtr &node, const std::string &file,
const std::unordered_map<std::string, std::string> &defines,
bool barebones) {
std::vector<StmtPtr> stmts;
auto preamble = std::make_shared<Preamble>();
if (!cache->module)
cache->module = new codon::ir::Module("", cache);
seqassert(cache->module, "cache's module is not set");
// Load standard library if it has not been loaded.
if (!in(cache->imports, STDLIB_IMPORT)) {
@ -89,7 +88,7 @@ StmtPtr SimplifyVisitor::apply(
}
// Reserve the following static identifiers.
for (auto name : {"staticlen", "compile_error", "isinstance", "hasattr", "type",
"TypeVar", "Callable"})
"TypeVar", "Callable", "argv"})
stdlib->generateCanonicalName(name);
// This code must be placed in a preamble (these are not POD types but are

12
codon/parser/visitors/simplify/simplify.h
View File

@ -27,9 +27,7 @@ namespace ast {
* ➡️ Note: This visitor *copies* the incoming AST and does not modify it.
*/
class SimplifyVisitor : public CallbackASTVisitor<ExprPtr, StmtPtr> {
/// Shared simplification context.
std::shared_ptr<SimplifyContext> ctx;
public:
/// Simplification step will divide the input AST into four sub-ASTs that are stored
/// here:
/// - Type (class) signatures
@ -47,6 +45,11 @@ class SimplifyVisitor : public CallbackASTVisitor<ExprPtr, StmtPtr> {
std::vector<StmtPtr> globals;
std::vector<StmtPtr> functions;
};
private:
/// Shared simplification context.
std::shared_ptr<SimplifyContext> ctx;
/// Preamble contains shared definition statements and is shared across all visitors
/// (in all modules). See Preamble (type) for more details.
std::shared_ptr<Preamble> preamble;
@ -69,8 +72,7 @@ public:
/// Each value is passed as a string (integer part is ignored).
/// The method will replace this map with a map that links canonical names
/// to their string and integer values.
static StmtPtr apply(std::shared_ptr<Cache> cache, const StmtPtr &node,
const std::string &file,
static StmtPtr apply(Cache *cache, const StmtPtr &node, const std::string &file,
const std::unordered_map<std::string, std::string> &defines,
bool barebones = false);

2
codon/parser/visitors/simplify/simplify_ctx.cpp
View File

@ -17,7 +17,7 @@ SimplifyItem::SimplifyItem(Kind k, std::string base, std::string canonicalName,
bool global)
: kind(k), base(move(base)), canonicalName(move(canonicalName)), global(global) {}
SimplifyContext::SimplifyContext(std::string filename, std::shared_ptr<Cache> cache)
SimplifyContext::SimplifyContext(std::string filename, Cache *cache)
: Context<SimplifyItem>(move(filename)), cache(move(cache)),
isStdlibLoading(false), moduleName{ImportFile::PACKAGE, "", ""}, canAssign(true),
allowTypeOf(true), substitutions(nullptr) {}

4
codon/parser/visitors/simplify/simplify_ctx.h
View File

@ -51,7 +51,7 @@ public:
*/
struct SimplifyContext : public Context<SimplifyItem> {
/// A pointer to the shared cache.
std::shared_ptr<Cache> cache;
Cache *cache;
/// A base scope definition. Each function or a class defines a new base scope.
struct Base {
@ -95,7 +95,7 @@ struct SimplifyContext : public Context<SimplifyItem> {
std::unordered_map<std::string, ExprPtr> *substitutions;
public:
SimplifyContext(std::string filename, std::shared_ptr<Cache> cache);
SimplifyContext(std::string filename, Cache *cache);
using Context<SimplifyItem>::add;
/// Convenience method for adding an object to the context.

17
codon/parser/visitors/simplify/simplify_stmt.cpp
View File

@ -156,7 +156,7 @@ void SimplifyVisitor::visit(AssertStmt *stmt) {
ExprPtr msg = N<StringExpr>("");
if (stmt->message)
msg = N<CallExpr>(N<IdExpr>("str"), clone(stmt->message));
if (ctx->getLevel() && ctx->bases.back().attributes & FLAG_TEST)
if (ctx->getLevel() && (ctx->bases.back().attributes & FLAG_TEST))
resultStmt = transform(
N<IfStmt>(N<UnaryExpr>("!", clone(stmt->expr)),
N<ExprStmt>(N<CallExpr>(N<DotExpr>("__internal__", "seq_assert_test"),
@ -301,7 +301,7 @@ void SimplifyVisitor::visit(ThrowStmt *stmt) {
}
void SimplifyVisitor::visit(WithStmt *stmt) {
assert(stmt->items.size());
seqassert(stmt->items.size(), "stmt->items is empty");
std::vector<StmtPtr> content;
for (int i = int(stmt->items.size()) - 1; i >= 0; i--) {
std::string var =
@ -328,7 +328,8 @@ void SimplifyVisitor::visit(GlobalStmt *stmt) {
error("not a top-level variable");
seqassert(!val->canonicalName.empty(), "'{}' does not have a canonical name",
stmt->var);
ctx->cache->globals.insert(val->canonicalName);
if (!in(ctx->cache->globals, val->canonicalName))
ctx->cache->globals[val->canonicalName] = nullptr;
val->global = true;
ctx->add(SimplifyItem::Var, stmt->var, val->canonicalName, true);
}
@ -1042,8 +1043,9 @@ StmtPtr SimplifyVisitor::transformAssignment(const ExprPtr &lhs, const ExprPtr &
// ctx->moduleName != MODULE_MAIN;
// ⚠️ TODO: should we make __main__ top-level variables NOT global by default?
// Problem: a = [1]; def foo(): a.append(2) won't work anymore as in Python.
if (global && !isStatic)
ctx->cache->globals.insert(canonical);
if (global && !isStatic && !(r && r->isType()) &&
!in(ctx->cache->globals, canonical))
ctx->cache->globals[canonical] = nullptr;
// Handle type aliases as well!
ctx->add(r && r->isType() ? SimplifyItem::Type : SimplifyItem::Var, e->value,
canonical, global);
@ -1336,7 +1338,8 @@ void SimplifyVisitor::transformNewImport(const ImportFile &file) {
// loaded)
preamble->globals.push_back(N<AssignStmt>(
N<IdExpr>(importDoneVar = importVar + "_done"), N<BoolExpr>(false)));
ctx->cache->globals.insert(importDoneVar);
if (!in(ctx->cache->globals, importDoneVar))
ctx->cache->globals[importDoneVar] = nullptr;
std::vector<StmtPtr> stmts;
stmts.push_back(nullptr); // placeholder to be filled later!
// We need to wrap all imported top-level statements (not signatures! they have
@ -1436,7 +1439,7 @@ StmtPtr SimplifyVisitor::transformLLVMDefinition(const Stmt *codeStmt) {
StmtPtr SimplifyVisitor::codegenMagic(const std::string &op, const Expr *typExpr,
const std::vector<Param> &args, bool isRecord) {
#define I(s) N<IdExpr>(s)
assert(typExpr);
seqassert(typExpr, "typExpr is null");
ExprPtr ret;
std::vector<Param> fargs;
std::vector<StmtPtr> stmts;

35
codon/parser/visitors/translate/translate.cpp
View File

@ -1,5 +1,6 @@
#include "translate.h"
#include <filesystem>
#include <memory>
#include <sstream>
#include <string>
@ -21,19 +22,34 @@ namespace ast {
TranslateVisitor::TranslateVisitor(std::shared_ptr<TranslateContext> ctx)
: ctx(std::move(ctx)), result(nullptr) {}
ir::Module *TranslateVisitor::apply(std::shared_ptr<Cache> cache, StmtPtr stmts) {
auto main = cast<ir::BodiedFunc>(cache->module->getMainFunc());
char buf[PATH_MAX + 1];
realpath(cache->module0.c_str(), buf);
main->setSrcInfo({std::string(buf), 0, 0, 0});
ir::Func *TranslateVisitor::apply(Cache *cache, StmtPtr stmts) {
ir::BodiedFunc *main;
if (cache->isJit) {
auto fnName = format("_jit_{}", cache->jitCell);
main = cache->module->Nr<ir::BodiedFunc>(fnName);
main->setSrcInfo({"<jit>", 0, 0, 0});
main->setGlobal();
auto irType = cache->module->unsafeGetFuncType(
fnName, cache->classes["void"].realizations["void"]->ir, {}, false);
main->realize(irType, {});
main->setJIT();
} else {
main = cast<ir::BodiedFunc>(cache->module->getMainFunc());
auto path =
std::filesystem::canonical(std::filesystem::path(cache->module0)).string();
main->setSrcInfo({path, 0, 0, 0});
}
auto block = cache->module->Nr<ir::SeriesFlow>("body");
main->setBody(block);
cache->codegenCtx = std::make_shared<TranslateContext>(cache, block, main);
if (!cache->codegenCtx)
cache->codegenCtx = std::make_shared<TranslateContext>(cache);
cache->codegenCtx->bases = {main};
cache->codegenCtx->series = {block};
TranslateVisitor(cache->codegenCtx).transform(stmts);
return cache->module;
return main;
}
/************************************************************************************/
@ -228,12 +244,15 @@ void TranslateVisitor::visit(AssignStmt *stmt) {
auto var = stmt->lhs->getId()->value;
if (!stmt->rhs && var == VAR_ARGV) {
ctx->add(TranslateItem::Var, var, ctx->getModule()->getArgVar());
ctx->cache->globals[var] = ctx->getModule()->getArgVar();
} else if (!stmt->rhs || !stmt->rhs->isType()) {
auto *newVar =
make<ir::Var>(stmt, getType((stmt->rhs ? stmt->rhs : stmt->lhs)->getType()),
in(ctx->cache->globals, var), var);
if (!in(ctx->cache->globals, var))
ctx->getBase()->push_back(newVar);
else
ctx->cache->globals[var] = newVar;
ctx->add(TranslateItem::Var, var, newVar);
if (stmt->rhs)
result = make<ir::AssignInstr>(stmt, newVar, transform(stmt->rhs));

2
codon/parser/visitors/translate/translate.h
View File

@ -22,7 +22,7 @@ class TranslateVisitor : public CallbackASTVisitor<ir::Value *, ir::Value *> {
public:
explicit TranslateVisitor(std::shared_ptr<TranslateContext> ctx);
static codon::ir::Module *apply(std::shared_ptr<Cache> cache, StmtPtr stmts);
static codon::ir::Func *apply(Cache *cache, StmtPtr stmts);
ir::Value *transform(const ExprPtr &expr) override;
ir::Value *transform(const StmtPtr &stmt) override;

6
codon/parser/visitors/translate/translate_ctx.cpp
View File

@ -11,13 +11,9 @@
namespace codon {
namespace ast {
TranslateContext::TranslateContext(std::shared_ptr<Cache> cache,
codon::ir::SeriesFlow *series,
codon::ir::BodiedFunc *base)
TranslateContext::TranslateContext(Cache *cache)
: Context<TranslateItem>(""), cache(std::move(cache)) {
stack.push_front(std::vector<std::string>());
bases.push_back(base);
addSeries(series);
}
std::shared_ptr<TranslateItem> TranslateContext::find(const std::string &name) const {

5
codon/parser/visitors/translate/translate_ctx.h
View File

@ -45,15 +45,14 @@ struct TranslateItem {
*/
struct TranslateContext : public Context<TranslateItem> {
/// A pointer to the shared cache.
std::shared_ptr<Cache> cache;
Cache *cache;
/// Stack of function bases.
std::vector<codon::ir::BodiedFunc *> bases;
/// Stack of IR series (blocks).
std::vector<codon::ir::SeriesFlow *> series;
public:
TranslateContext(std::shared_ptr<Cache> cache, codon::ir::SeriesFlow *series,
codon::ir::BodiedFunc *base);
TranslateContext(Cache *cache);
using Context<TranslateItem>::add;
/// Convenience method for adding an object to the context.

10
codon/parser/visitors/typecheck/typecheck.cpp
View File

@ -23,10 +23,12 @@ TypecheckVisitor::TypecheckVisitor(std::shared_ptr<TypeContext> ctx,
prependStmts = stmts ? stmts : std::make_shared<std::vector<StmtPtr>>();
}
StmtPtr TypecheckVisitor::apply(std::shared_ptr<Cache> cache, StmtPtr stmts) {
auto ctx = std::make_shared<TypeContext>(cache);
cache->typeCtx = ctx;
TypecheckVisitor v(ctx);
StmtPtr TypecheckVisitor::apply(Cache *cache, StmtPtr stmts) {
if (!cache->typeCtx) {
auto ctx = std::make_shared<TypeContext>(cache);
cache->typeCtx = ctx;
}
TypecheckVisitor v(cache->typeCtx);
auto infer = v.inferTypes(stmts->clone(), true, "<top>");
return std::move(infer.second);
}

2
codon/parser/visitors/typecheck/typecheck.h
View File

@ -26,7 +26,7 @@ class TypecheckVisitor : public CallbackASTVisitor<ExprPtr, StmtPtr> {
StmtPtr resultStmt;
public:
static StmtPtr apply(std::shared_ptr<Cache> cache, StmtPtr stmts);
static StmtPtr apply(Cache *cache, StmtPtr stmts);
public:
explicit TypecheckVisitor(

8
codon/parser/visitors/typecheck/typecheck_ctx.cpp
View File

@ -14,7 +14,7 @@ using fmt::format;
namespace codon {
namespace ast {
TypeContext::TypeContext(std::shared_ptr<Cache> cache)
TypeContext::TypeContext(Cache *cache)
: Context<TypecheckItem>(""), cache(move(cache)), typecheckLevel(0),
allowActivation(true), age(0), realizationDepth(0) {
stack.push_front(std::vector<std::string>());
@ -89,7 +89,7 @@ TypeContext::addUnbound(const Expr *expr, int level, bool setActive, char static
types::TypePtr TypeContext::instantiate(const Expr *expr, types::TypePtr type,
types::ClassType *generics, bool activate) {
assert(type);
seqassert(type, "type is null");
std::unordered_map<int, types::TypePtr> genericCache;
if (generics)
for (auto &g : generics->generics)
@ -122,12 +122,12 @@ types::TypePtr
TypeContext::instantiateGeneric(const Expr *expr, types::TypePtr root,
const std::vector<types::TypePtr> &generics) {
auto c = root->getClass();
assert(c);
seqassert(c, "root class is null");
auto g = std::make_shared<types::ClassType>("", ""); // dummy generic type
if (generics.size() != c->generics.size())
error(expr->getSrcInfo(), "generics do not match");
for (int i = 0; i < c->generics.size(); i++) {
assert(c->generics[i].type);
seqassert(c->generics[i].type, "generic is null");
g->generics.push_back(
types::ClassType::Generic("", "", generics[i], c->generics[i].id));
}

4
codon/parser/visitors/typecheck/typecheck_ctx.h
View File

@ -33,7 +33,7 @@ struct TypecheckItem {
*/
struct TypeContext : public Context<TypecheckItem> {
/// A pointer to the shared cache.
std::shared_ptr<Cache> cache;
Cache *cache;
/// A realization base definition. Each function realization defines a new base scope.
/// Used to properly realize enclosed functions and to prevent mess with mutually
@ -68,7 +68,7 @@ struct TypeContext : public Context<TypecheckItem> {
std::set<std::string> defaultCallDepth;
public:
explicit TypeContext(std::shared_ptr<Cache> cache);
explicit TypeContext(Cache *cache);
using Context<TypecheckItem>::add;
/// Convenience method for adding an object to the context.

5
codon/parser/visitors/typecheck/typecheck_expr.cpp
View File

@ -1143,8 +1143,9 @@ ExprPtr TypecheckVisitor::transformCall(CallExpr *expr, const types::TypePtr &in
// Typecheck given arguments with the expected (signature) types.
bool unificationsDone = true;
assert((expr->ordered && typeArgs.empty()) ||
(!expr->ordered && typeArgs.size() == calleeFn->funcGenerics.size()));
seqassert((expr->ordered && typeArgs.empty()) ||
(!expr->ordered && typeArgs.size() == calleeFn->funcGenerics.size()),
"bad vector sizes");
for (int si = 0; !expr->ordered && si < calleeFn->funcGenerics.size(); si++)
if (typeArgs[si]) {
auto t = typeArgs[si]->type;

26
codon/parser/visitors/typecheck/typecheck_infer.cpp
View File

@ -1,3 +1,4 @@
#include <limits>
#include <map>
#include <memory>
#include <string>
@ -149,7 +150,7 @@ types::TypePtr TypecheckVisitor::realizeFunc(types::FuncType *type) {
LOG_REALIZE("[realize] fn {} -> {} : base {} ; depth = {}", type->ast->name,
type->realizedName(), ctx->getBase(), depth);
{
_level++;
getLogger().level++;
ctx->realizationDepth++;
ctx->addBlock();
ctx->typecheckLevel++;
@ -205,7 +206,7 @@ types::TypePtr TypecheckVisitor::realizeFunc(types::FuncType *type) {
// Realize the return type.
if (auto t = realize(type->args[0]))
unify(type->args[0], t);
LOG_TYPECHECK("done with {} / {}", type->realizedName(), oldKey);
LOG_REALIZE("done with {} / {}", type->realizedName(), oldKey);
// Create and store IR node and a realized AST to be used
// during the code generation.
@ -220,8 +221,6 @@ types::TypePtr TypecheckVisitor::realizeFunc(types::FuncType *type) {
r->ir = ctx->cache->module->Nr<ir::ExternalFunc>(type->realizedName());
} else {
r->ir = ctx->cache->module->Nr<ir::BodiedFunc>(type->realizedName());
if (ast->attributes.has(Attr::ForceRealize))
ir::cast<ir::BodiedFunc>(r->ir)->setBuiltin();
}
auto parent = type->funcParent;
@ -257,7 +256,7 @@ types::TypePtr TypecheckVisitor::realizeFunc(types::FuncType *type) {
ctx->popBlock();
ctx->typecheckLevel--;
ctx->realizationDepth--;
_level--;
getLogger().level--;
}
// Restore old bases back.
ctx->bases.insert(ctx->bases.end(), oldBases.begin(), oldBases.end());
@ -280,7 +279,7 @@ std::pair<int, StmtPtr> TypecheckVisitor::inferTypes(StmtPtr result, bool keepLa
int minUnbound = ctx->cache->unboundCount;
ctx->addBlock();
int iteration = 0;
for (int prevSize = INT_MAX;;) {
for (int prevSize = std::numeric_limits<int>::max();;) {
LOG_TYPECHECK("== iter {} ==========================================", iteration);
ctx->typecheckLevel++;
result = TypecheckVisitor(ctx).transform(result);
@ -306,7 +305,7 @@ std::pair<int, StmtPtr> TypecheckVisitor::inferTypes(StmtPtr result, bool keepLa
std::map<types::TypePtr, std::string> newActiveUnbounds;
for (auto i = ctx->activeUnbounds.begin(); i != ctx->activeUnbounds.end();) {
auto l = i->first->getLink();
assert(l);
seqassert(l, "link is null");
if (l->kind == LinkType::Unbound) {
newActiveUnbounds[i->first] = i->second;
if (l->id >= minUnbound)
@ -405,20 +404,21 @@ ir::types::Type *TypecheckVisitor::getLLVMType(const types::ClassType *t) {
} else if (name == "str") {
handle = module->getStringType();
} else if (name == "Int" || name == "UInt") {
assert(statics.size() == 1 && statics[0]->type == StaticValue::INT &&
types.empty());
seqassert(statics.size() == 1 && statics[0]->type == StaticValue::INT &&
types.empty(),
"bad generics/statics");
handle = module->Nr<ir::types::IntNType>(statics[0]->getInt(), name == "Int");
} else if (name == "Ptr") {
assert(types.size() == 1 && statics.empty());
seqassert(types.size() == 1 && statics.empty(), "bad generics/statics");
handle = module->unsafeGetPointerType(types[0]);
} else if (name == "Generator") {
assert(types.size() == 1 && statics.empty());
seqassert(types.size() == 1 && statics.empty(), "bad generics/statics");
handle = module->unsafeGetGeneratorType(types[0]);
} else if (name == TYPE_OPTIONAL) {
assert(types.size() == 1 && statics.empty());
seqassert(types.size() == 1 && statics.empty(), "bad generics/statics");
handle = module->unsafeGetOptionalType(types[0]);
} else if (name == "NoneType") {
assert(types.empty() && statics.empty());
seqassert(types.empty() && statics.empty(), "bad generics/statics");
auto record =
ir::cast<ir::types::RecordType>(module->unsafeGetMemberedType(realizedName));
record->realize({}, {});

121
codon/runtime/exc.cpp
View File

@ -7,6 +7,8 @@
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <mutex>
#include <sstream>
#include <string>
#include <vector>
@ -41,6 +43,8 @@ struct Backtrace {
frames[count++] = {functionDup, filenameDup, pc, lineno};
}
void push_back(uintptr_t pc) { push_back("<invalid>", "<invalid>", pc, 0); }
void free() {
for (auto i = 0; i < count; i++) {
auto *frame = &frames[i];
@ -54,7 +58,7 @@ struct Backtrace {
};
void seq_backtrace_error_callback(void *data, const char *msg, int errnum) {
// nothing to do
// printf("seq_backtrace_error_callback: %s (errnum = %d)\n", msg, errnum);
}
int seq_backtrace_full_callback(void *data, uintptr_t pc, const char *filename,
@ -64,6 +68,12 @@ int seq_backtrace_full_callback(void *data, uintptr_t pc, const char *filename,
return (bt->count < Backtrace::LIMIT) ? 0 : 1;
}
int seq_backtrace_simple_callback(void *data, uintptr_t pc) {
auto *bt = ((Backtrace *)data);
bt->push_back(pc);
return (bt->count < Backtrace::LIMIT) ? 0 : 1;
}
/*
* This is largely based on
* llvm/examples/ExceptionDemo/ExceptionDemo.cpp
@ -127,7 +137,7 @@ static void seq_delete_exc(_Unwind_Exception *expToDelete) {
if (!expToDelete || expToDelete->exception_class != ourBaseExceptionClass)
return;
auto *exc = (OurException *)((char *)expToDelete + ourBaseFromUnwindOffset);
if (seq_debug) {
if (seq_flags & SEQ_FLAG_DEBUG) {
exc->bt.free();
}
seq_free(exc);
@ -139,6 +149,7 @@ static void seq_delete_unwind_exc(_Unwind_Reason_Code reason,
}
static struct backtrace_state *state = nullptr;
static std::mutex stateLock;
SEQ_FUNC void *seq_alloc_exc(int type, void *obj) {
const size_t size = sizeof(OurException);
@ -148,19 +159,30 @@ SEQ_FUNC void *seq_alloc_exc(int type, void *obj) {
e->obj = obj;
e->unwindException.exception_class = ourBaseExceptionClass;
e->unwindException.exception_cleanup = seq_delete_unwind_exc;
if (seq_debug) {
if (seq_flags & SEQ_FLAG_DEBUG) {
e->bt.frames = nullptr;
e->bt.count = 0;
if (!state)
state = backtrace_create_state(/*filename=*/nullptr, /*threaded=*/0,
if (seq_flags & SEQ_FLAG_STANDALONE) {
if (!state) {
stateLock.lock();
if (!state)
state =
backtrace_create_state(/*filename=*/nullptr, /*threaded=*/1,
seq_backtrace_error_callback, /*data=*/nullptr);
backtrace_full(state, /*skip=*/1, seq_backtrace_full_callback,
seq_backtrace_error_callback, &e->bt);
stateLock.unlock();
}
backtrace_full(state, /*skip=*/1, seq_backtrace_full_callback,
seq_backtrace_error_callback, &e->bt);
} else {
backtrace_simple(/*state=*/nullptr, /*skip=*/1, seq_backtrace_simple_callback,
seq_backtrace_error_callback, &e->bt);
}
}
return &(e->unwindException);
}
static void print_from_last_dot(seq_str_t s) {
static void print_from_last_dot(seq_str_t s, std::ostringstream &buf) {
char *p = s.str;
int64_t n = s.len;
@ -172,7 +194,7 @@ static void print_from_last_dot(seq_str_t s) {
}
}
fwrite(p, 1, (size_t)n, stderr);
buf.write(p, (size_t)n);
}
SEQ_FUNC void seq_terminate(void *exc) {
@ -186,41 +208,64 @@ SEQ_FUNC void seq_terminate(void *exc) {
exit((int)status);
}
fprintf(stderr, "\033[1m");
print_from_last_dot(hdr->type);
std::ostringstream buf;
if (seq_flags & SEQ_FLAG_JIT)
buf << codon::getCapturedOutput();
buf << "\033[1m";
print_from_last_dot(hdr->type, buf);
if (hdr->msg.len > 0) {
fprintf(stderr, ": ");
fprintf(stderr, "\033[0m");
fwrite(hdr->msg.str, 1, (size_t)hdr->msg.len, stderr);
buf << ": \033[0m";
buf.write(hdr->msg.str, hdr->msg.len);
} else {
fprintf(stderr, "\033[0m");
buf << "\033[0m";
}
fprintf(stderr, "\n\n");
fprintf(stderr, "\033[1mRaised from:\033[0m \033[32m");
fwrite(hdr->func.str, 1, (size_t)hdr->func.len, stderr);
fprintf(stderr, "\033[0m\n");
fwrite(hdr->file.str, 1, (size_t)hdr->file.len, stderr);
buf << "\n\n\033[1mRaised from:\033[0m \033[32m";
buf.write(hdr->func.str, hdr->func.len);
buf << "\033[0m\n";
buf.write(hdr->file.str, hdr->file.len);
if (hdr->line > 0) {
fprintf(stderr, ":%lld", (long long)hdr->line);
buf << ":" << hdr->line;
if (hdr->col > 0)
fprintf(stderr, ":%lld", (long long)hdr->col);
buf << ":" << hdr->col;
}
fprintf(stderr, "\n");
buf << "\n";
if (seq_debug) {
if ((seq_flags & SEQ_FLAG_DEBUG) && (seq_flags & SEQ_FLAG_STANDALONE)) {
auto *bt = &base->bt;
if (bt->count > 0) {
fprintf(stderr, "\n\033[1mBacktrace:\033[0m\n");
buf << "\n\033[1mBacktrace:\033[0m\n";
for (unsigned i = 0; i < bt->count; i++) {
auto *frame = &bt->frames[i];
fprintf(stderr, " [\033[33m0x%lx\033[0m] \033[32m%s\033[0m %s:%d\n", frame->pc,
frame->function, frame->filename, frame->lineno);
buf << " "
<< codon::makeBacktraceFrameString(frame->pc, std::string(frame->function),
std::string(frame->filename),
frame->lineno)
<< "\n";
}
}
}
abort();
auto output = buf.str();
if (seq_flags & SEQ_FLAG_STANDALONE) {
fwrite(output.data(), 1, output.size(), stderr);
abort();
} else {
auto *bt = &base->bt;
std::string msg(hdr->msg.str, hdr->msg.len);
std::string file(hdr->file.str, hdr->file.len);
std::string type(hdr->type.str, hdr->type.len);
std::vector<uintptr_t> backtrace;
if (seq_flags & SEQ_FLAG_DEBUG) {
for (unsigned i = 0; i < bt->count; i++) {
backtrace.push_back(bt->frames[i].pc);
}
}
throw codon::JITError(output, msg, type, file, (int)hdr->line, (int)hdr->col,
backtrace);
}
}
SEQ_FUNC void seq_throw(void *exc) {
@ -546,3 +591,23 @@ SEQ_FUNC int64_t seq_exc_offset() {
SEQ_FUNC uint64_t seq_exc_class() {
return genClass(ourBaseExcpClassChars, sizeof(ourBaseExcpClassChars));
}
std::string codon::makeBacktraceFrameString(uintptr_t pc, const std::string &func,
const std::string &file, int line,
int col) {
std::ostringstream buf;
buf << "[\033[33m0x" << std::hex << pc << std::dec << "\033[0m]";
if (!func.empty()) {
buf << " \033[32m" << func << "\033[0m";
if (!file.empty()) {
buf << " at \033[36m" << file << "\033[0m";
if (line != 0) {
buf << ":\033[33m" << line << "\033[0m";
if (col != 0) {
buf << ":\033[33m" << col << "\033[0m";
}
}
}
}
return buf.str();
}

26
codon/runtime/lib.cpp
View File

@ -10,6 +10,7 @@
#include <fstream>
#include <iostream>
#include <mutex>
#include <sstream>
#include <string>
#include <unistd.h>
#include <unwind.h>
@ -45,16 +46,16 @@ static void register_thread(kmp_int32 *global_tid, kmp_int32 *bound_tid) {
void seq_exc_init();
int seq_debug;
int seq_flags;
SEQ_FUNC void seq_init(int d) {
SEQ_FUNC void seq_init(int flags) {
GC_INIT();
GC_set_warn_proc(GC_ignore_warn_proc);
GC_allow_register_threads();
// equivalent to: #pragma omp parallel { register_thread }
__kmpc_fork_call(&dummy_loc, 0, (kmpc_micro)register_thread);
seq_exc_init();
seq_debug = d;
seq_flags = flags;
}
SEQ_FUNC bool seq_is_macos() {
@ -214,10 +215,25 @@ SEQ_FUNC seq_str_t seq_check_errno() {
return {0, nullptr};
}
SEQ_FUNC void seq_print(seq_str_t str) { fwrite(str.str, 1, (size_t)str.len, stdout); }
SEQ_FUNC void seq_print(seq_str_t str) { seq_print_full(str, stdout); }
static std::ostringstream capture;
static std::mutex captureLock;
SEQ_FUNC void seq_print_full(seq_str_t str, FILE *fo) {
fwrite(str.str, 1, (size_t)str.len, fo);
if ((seq_flags & SEQ_FLAG_JIT) && (fo == stdout || fo == stderr)) {
captureLock.lock();
capture.write(str.str, str.len);
captureLock.unlock();
} else {
fwrite(str.str, 1, (size_t)str.len, fo);
}
}
std::string codon::getCapturedOutput() {
std::string result = capture.str();
capture.str("");
return result;
}
SEQ_FUNC void *seq_stdin() { return stdin; }

51
codon/runtime/lib.h
View File

@ -1,29 +1,30 @@
#ifndef CODON_RUNTIME_LIB_H
#define CODON_RUNTIME_LIB_H
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <stdexcept>
#include <string>
#include <unwind.h>
#include <vector>
#define SEQ_FLAG_DEBUG (1 << 0) // compiled/running in debug mode
#define SEQ_FLAG_JIT (1 << 1) // compiled/running in JIT mode
#define SEQ_FLAG_STANDALONE (1 << 2) // compiled as a standalone object/binary
#define SEQ_FUNC extern "C"
typedef int64_t seq_int_t;
struct seq_t {
seq_int_t len;
char *seq;
};
struct seq_str_t {
seq_int_t len;
char *str;
};
extern int seq_debug;
extern int seq_flags;
SEQ_FUNC void seq_init(int debug);
SEQ_FUNC void seq_init(int flags);
SEQ_FUNC bool seq_is_macos();
SEQ_FUNC seq_int_t seq_pid();
@ -76,4 +77,34 @@ SEQ_FUNC void *seq_rlock_new();
SEQ_FUNC bool seq_rlock_acquire(void *lock, bool block, double timeout);
SEQ_FUNC void seq_rlock_release(void *lock);
#endif /* CODON_RUNTIME_LIB_H */
namespace codon {
class JITError : public std::runtime_error {
private:
std::string output;
std::string type;
std::string file;
int line;
int col;
std::vector<uintptr_t> backtrace;
public:
JITError(const std::string &output, const std::string &what, const std::string &type,
const std::string &file, int line, int col,
std::vector<uintptr_t> backtrace = {})
: std::runtime_error(what), output(output), type(type), file(file), line(line),
col(col), backtrace(std::move(backtrace)) {}
std::string getOutput() const { return output; }
std::string getType() const { return type; }
std::string getFile() const { return file; }
int getLine() const { return line; }
int getCol() const { return col; }
std::vector<uintptr_t> getBacktrace() const { return backtrace; }
};
std::string makeBacktraceFrameString(uintptr_t pc, const std::string &func = "",
const std::string &file = "", int line = 0,
int col = 0);
std::string getCapturedOutput();
} // namespace codon

14
codon/sir/func.h
View File

@ -88,8 +88,8 @@ private:
std::list<Var *> symbols;
/// the function body
Value *body = nullptr;
/// whether the function is builtin
bool builtin = false;
/// whether the function is a JIT input
bool jit = false;
public:
static const char NodeId;
@ -136,11 +136,11 @@ public:
/// @param b the new body
void setBody(Flow *b) { body = b; }
/// @return true if the function is builtin
bool isBuiltin() const { return builtin; }
/// Changes the function's builtin status.
/// @param v true if builtin, false otherwise
void setBuiltin(bool v = true) { builtin = v; }
/// @return true if the function is a JIT input
bool isJIT() const { return jit; }
/// Changes the function's JIT input status.
/// @param v true if JIT input, false otherwise
void setJIT(bool v = true) { jit = v; }
protected:
std::vector<Value *> doGetUsedValues() const override {

1375
codon/sir/llvm/llvisitor.cpp
View File

File diff suppressed because it is too large Load Diff

156
codon/sir/llvm/llvisitor.h
View File

@ -3,9 +3,11 @@
#include "codon/dsl/plugins.h"
#include "codon/sir/llvm/llvm.h"
#include "codon/sir/sir.h"
#include "codon/util/common.h"
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace codon {
@ -13,19 +15,6 @@ namespace ir {
class LLVMVisitor : public util::ConstVisitor {
private:
template <typename V> using CacheBase = std::unordered_map<id_t, V *>;
template <typename K, typename V> class Cache : public CacheBase<V> {
public:
using CacheBase<V>::CacheBase;
V *operator[](const K *key) {
auto it = CacheBase<V>::find(key->getId());
return (it != CacheBase<V>::end()) ? it->second : nullptr;
}
void insert(const K *key, V *value) { CacheBase<V>::emplace(key->getId(), value); }
};
struct CoroData {
/// Coroutine promise (where yielded values are stored)
llvm::Value *promise;
@ -37,6 +26,8 @@ private:
llvm::BasicBlock *suspend;
/// Coroutine exit block
llvm::BasicBlock *exit;
void reset() { promise = handle = cleanup = suspend = exit = nullptr; }
};
struct NestableData {
@ -56,6 +47,8 @@ private:
LoopData(llvm::BasicBlock *breakBlock, llvm::BasicBlock *continueBlock, id_t loopId)
: NestableData(), breakBlock(breakBlock), continueBlock(continueBlock),
loopId(loopId) {}
void reset() { breakBlock = continueBlock = nullptr; }
};
struct TryCatchData : NestableData {
@ -87,6 +80,13 @@ private:
finallyBlock(nullptr), catchTypes(), handlers(), excFlag(nullptr),
catchStore(nullptr), delegateDepth(nullptr), retStore(nullptr),
loopSequence(nullptr) {}
void reset() {
exceptionBlock = exceptionRouteBlock = finallyBlock = nullptr;
catchTypes.clear();
handlers.clear();
excFlag = catchStore = delegateDepth = loopSequence = nullptr;
}
};
struct DebugInfo {
@ -96,21 +96,31 @@ private:
llvm::DICompileUnit *unit;
/// Whether we are compiling in debug mode
bool debug;
/// Whether we are compiling in JIT mode
bool jit;
/// Whether we are compiling a standalone object/executable
bool standalone;
/// Program command-line flags
std::string flags;
explicit DebugInfo(bool debug, const std::string &flags)
: builder(), unit(nullptr), debug(debug), flags(flags) {}
DebugInfo()
: builder(), unit(nullptr), debug(false), jit(false), standalone(false),
flags() {}
llvm::DIFile *getFile(const std::string &path);
void reset() {
builder = {};
unit = nullptr;
}
};
/// LLVM context used for compilation
llvm::LLVMContext context;
/// LLVM IR builder used for constructing LLVM IR
llvm::IRBuilder<> builder;
std::unique_ptr<llvm::LLVMContext> context;
/// Module we are compiling
std::unique_ptr<llvm::Module> module;
std::unique_ptr<llvm::Module> M;
/// LLVM IR builder used for constructing LLVM IR
std::unique_ptr<llvm::IRBuilder<>> B;
/// Current function we are compiling
llvm::Function *func;
/// Current basic block we are compiling
@ -118,9 +128,9 @@ private:
/// Last compiled value
llvm::Value *value;
/// LLVM values corresponding to IR variables
Cache<Var, llvm::Value> vars;
std::unordered_map<id_t, llvm::Value *> vars;
/// LLVM functions corresponding to IR functions
Cache<Func, llvm::Function> funcs;
std::unordered_map<id_t, llvm::Function *> funcs;
/// Coroutine data, if current function is a coroutine
CoroData coro;
/// Loop data stack, containing break/continue blocks
@ -175,17 +185,82 @@ private:
// LLVM passes
void runLLVMPipeline();
public:
LLVMVisitor(bool debug = false, const std::string &flags = "");
llvm::Value *getVar(const Var *var);
void insertVar(const Var *var, llvm::Value *x) { vars.emplace(var->getId(), x); }
llvm::Function *getFunc(const Func *func);
void insertFunc(const Func *func, llvm::Function *x) {
funcs.emplace(func->getId(), x);
}
llvm::Value *getDummyVoidValue() { return llvm::ConstantTokenNone::get(*context); }
llvm::DISubprogram *getDISubprogramForFunc(const Func *x);
llvm::LLVMContext &getContext() { return context; }
llvm::IRBuilder<> &getBuilder() { return builder; }
llvm::Module *getModule() { return module.get(); }
public:
static std::string getNameForFunction(const Func *x) {
if (isA<ExternalFunc>(x)) {
return x->getUnmangledName();
} else {
return x->referenceString();
}
}
static std::string getDebugNameForVariable(const Var *x) {
std::string name = x->getName();
auto pos = name.find(".");
if (pos != 0 && pos != std::string::npos) {
return name.substr(0, pos);
} else {
return name;
}
}
static const SrcInfo *getDefaultSrcInfo() {
static SrcInfo defaultSrcInfo("<internal>", 0, 0, 0);
return &defaultSrcInfo;
}
static const SrcInfo *getSrcInfo(const Node *x) {
if (auto *srcInfo = x->getAttribute<SrcInfoAttribute>()) {
return &srcInfo->info;
} else {
return getDefaultSrcInfo();
}
}
/// Constructs an LLVM visitor.
LLVMVisitor();
/// @return true if in debug mode, false otherwise
bool getDebug() const { return db.debug; }
/// Sets debug status.
/// @param d true if debug mode
void setDebug(bool d = true) { db.debug = d; }
/// @return true if in JIT mode, false otherwise
bool getJIT() const { return db.jit; }
/// Sets JIT status.
/// @param j true if JIT mode
void setJIT(bool j = true) { db.jit = j; }
/// @return true if in standalone mode, false otherwise
bool getStandalone() const { return db.standalone; }
/// Sets standalone status.
/// @param s true if standalone
void setStandalone(bool s = true) { db.standalone = s; }
/// @return program flags
std::string getFlags() const { return db.flags; }
/// Sets program flags.
/// @param f flags
void setFlags(const std::string &f) { db.flags = f; }
llvm::LLVMContext &getContext() { return *context; }
llvm::IRBuilder<> &getBuilder() { return *B; }
llvm::Module *getModule() { return M.get(); }
llvm::FunctionCallee getFunc() { return func; }
llvm::BasicBlock *getBlock() { return block; }
llvm::Value *getValue() { return value; }
Cache<Var, llvm::Value> &getVars() { return vars; }
Cache<Func, llvm::Function> &getFuncs() { return funcs; }
std::unordered_map<id_t, llvm::Value *> &getVars() { return vars; }
std::unordered_map<id_t, llvm::Function *> &getFuncs() { return funcs; }
CoroData &getCoro() { return coro; }
std::vector<LoopData> &getLoops() { return loops; }
std::vector<TryCatchData> &getTryCatch() { return trycatch; }
@ -195,6 +270,31 @@ public:
void setBlock(llvm::BasicBlock *b) { block = b; }
void setValue(llvm::Value *v) { value = v; }
/// Registers a new global variable or function with
/// this visitor.
/// @param var the global variable (or function) to register
void registerGlobal(const Var *var);
/// Returns the default LLVM linkage type for the module.
/// @return LLVM linkage type
llvm::GlobalValue::LinkageTypes getDefaultLinkage();
/// Returns a new LLVM module initialized for the host
/// architecture.
/// @param context LLVM context used for creating module
/// @param src source information for the new module
/// @return a new module
std::unique_ptr<llvm::Module> makeModule(llvm::LLVMContext &context,
const SrcInfo *src = nullptr);
/// Returns the current module/LLVM context and replaces them
/// with new, fresh ones. References to variables or functions
/// from the old module will be included as "external".
/// @param src source information for the new module
/// @return the current module/context, replaced internally
std::pair<std::unique_ptr<llvm::Module>, std::unique_ptr<llvm::LLVMContext>>
takeModule(const SrcInfo *src = nullptr);
/// Sets current debug info based on a given node.
/// @param node the node whose debug info to use
void setDebugInfoForNode(const Node *node);

2
codon/sir/llvm/llvm.h
View File

@ -12,8 +12,10 @@
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/DebugInfo/Symbolize/Symbolize.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/JITEventListener.h"
#include "llvm/ExecutionEngine/JITSymbol.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/ExecutionEngine/RuntimeDyld.h"

40
codon/sir/llvm/optimize.cpp
View File

@ -1,7 +1,5 @@
#include "optimize.h"
#include <chrono>
#include "codon/sir/llvm/coro/Coroutines.h"
#include "codon/util/common.h"
#include "llvm/CodeGen/CommandFlags.h"
@ -51,15 +49,14 @@ std::unique_ptr<llvm::TargetMachine> getTargetMachine(llvm::Module *module,
}
namespace {
void applyDebugTransformations(llvm::Module *module, bool debug) {
void applyDebugTransformations(llvm::Module *module, bool debug, bool jit) {
if (debug) {
// remove tail calls and fix linkage for stack traces
for (auto &f : *module) {
f.setLinkage(llvm::GlobalValue::ExternalLinkage);
if (f.hasFnAttribute(llvm::Attribute::AttrKind::AlwaysInline)) {
f.removeFnAttr(llvm::Attribute::AttrKind::AlwaysInline);
}
f.addFnAttr(llvm::Attribute::AttrKind::NoInline);
if (!jit)
f.setLinkage(llvm::GlobalValue::ExternalLinkage);
if (!f.hasFnAttribute(llvm::Attribute::AttrKind::AlwaysInline))
f.addFnAttr(llvm::Attribute::AttrKind::NoInline);
f.setHasUWTable();
f.addFnAttr("no-frame-pointer-elim", "true");
f.addFnAttr("no-frame-pointer-elim-non-leaf");
@ -163,9 +160,9 @@ char CoroBranchSimplifier::ID = 0;
llvm::RegisterPass<CoroBranchSimplifier> X("coro-br-simpl",
"Coroutine Branch Simplifier");
void runLLVMOptimizationPasses(llvm::Module *module, bool debug,
void runLLVMOptimizationPasses(llvm::Module *module, bool debug, bool jit,
PluginManager *plugins) {
applyDebugTransformations(module, debug);
applyDebugTransformations(module, debug, jit);
llvm::Triple moduleTriple(module->getTargetTriple());
llvm::TargetLibraryInfoImpl tlii(moduleTriple);
@ -227,7 +224,7 @@ void runLLVMOptimizationPasses(llvm::Module *module, bool debug,
}
fpm->doFinalization();
pm->run(*module);
applyDebugTransformations(module, debug);
applyDebugTransformations(module, debug, jit);
}
void verify(llvm::Module *module) {
@ -237,22 +234,15 @@ void verify(llvm::Module *module) {
} // namespace
void optimize(llvm::Module *module, bool debug, PluginManager *plugins) {
using std::chrono::duration_cast;
using std::chrono::high_resolution_clock;
using std::chrono::milliseconds;
auto t = high_resolution_clock::now();
void optimize(llvm::Module *module, bool debug, bool jit, PluginManager *plugins) {
verify(module);
runLLVMOptimizationPasses(module, debug, plugins);
LOG_TIME("[T] llvm/opt = {:.1f}",
duration_cast<milliseconds>(high_resolution_clock::now() - t).count() /
1000.0);
{
TIME("llvm/opt");
runLLVMOptimizationPasses(module, debug, jit, plugins);
}
if (!debug) {
t = high_resolution_clock::now();
runLLVMOptimizationPasses(module, debug, plugins);
LOG_TIME("[T] llvm/opt2 = {:.1f}",
duration_cast<milliseconds>(high_resolution_clock::now() - t).count() /
1000.0);
TIME("llvm/opt2");
runLLVMOptimizationPasses(module, debug, jit, plugins);
}
verify(module);
}

3
codon/sir/llvm/optimize.h
View File

@ -14,6 +14,7 @@ getTargetMachine(llvm::Triple triple, llvm::StringRef cpuStr,
std::unique_ptr<llvm::TargetMachine>
getTargetMachine(llvm::Module *module, bool setFunctionAttributes = false);
void optimize(llvm::Module *module, bool debug, PluginManager *plugins = nullptr);
void optimize(llvm::Module *module, bool debug, bool jit = false,
PluginManager *plugins = nullptr);
} // namespace ir
} // namespace codon

3
codon/sir/module.cpp
View File

@ -100,8 +100,7 @@ const std::string Module::INIT_MAGIC_NAME = "__init__";
const char Module::NodeId = 0;
Module::Module(std::string name, std::shared_ptr<ast::Cache> cache)
: AcceptorExtend(std::move(name)), cache(std::move(cache)) {
Module::Module(const std::string &name) : AcceptorExtend(name) {
mainFunc = std::make_unique<BodiedFunc>("main");
mainFunc->realize(cast<types::FuncType>(unsafeGetDummyFuncType()), {});
mainFunc->setModule(this);

12
codon/sir/module.h
View File

@ -92,15 +92,14 @@ private:
typesMap;
/// the type-checker cache
std::shared_ptr<ast::Cache> cache;
ast::Cache *cache = nullptr;
public:
static const char NodeId;
/// Constructs an SIR module.
/// @param name the module name
/// @param cache the type-checker cache
explicit Module(std::string name, std::shared_ptr<ast::Cache> cache = nullptr);
explicit Module(const std::string &name = "");
virtual ~Module() noexcept = default;
@ -259,9 +258,10 @@ public:
}
/// @return the type-checker cache
std::shared_ptr<ast::Cache> getCache() { return cache; }
/// @return the type-checker cache
std::shared_ptr<const ast::Cache> getCache() const { return cache; }
ast::Cache *getCache() const { return cache; }
/// Sets the type-checker cache.
/// @param c the cache
void setCache(ast::Cache *c) { cache = c; }
/// Gets or realizes a method.
/// @param parent the parent class

2
codon/sir/transform/cleanup/global_demote.cpp
View File

@ -45,7 +45,7 @@ void GlobalDemotionPass::run(Module *M) {
}
for (auto it : localGlobals) {
if (!it.second)
if (!it.second || it.first->getId() == M->getArgVar()->getId())
continue;
seqassert(it.first->isGlobal(), "var was not global");
it.first->setGlobal(false);

1
codon/sir/transform/folding/const_prop.h
View File

@ -20,6 +20,7 @@ public:
/// Constructs a constant propagation pass.
/// @param reachingDefKey the reaching definition analysis' key
/// @param globalVarsKey global variables analysis' key
ConstPropPass(const std::string &reachingDefKey, const std::string &globalVarsKey)
: reachingDefKey(reachingDefKey), globalVarsKey(globalVarsKey) {}

5
codon/sir/transform/manager.cpp
View File

@ -1,6 +1,5 @@
#include "manager.h"
#include <cassert>
#include <unordered_set>
#include "codon/sir/analyze/analysis.h"
@ -49,7 +48,7 @@ std::string PassManager::registerPass(std::unique_ptr<Pass> pass,
key = km.getUniqueKey(key);
for (const auto &req : reqs) {
assert(deps.find(req) != deps.end());
seqassert(deps.find(req) != deps.end(), "required key '{}' not found", req);
deps[req].push_back(key);
}
@ -76,7 +75,7 @@ std::string PassManager::registerAnalysis(std::unique_ptr<analyze::Analysis> ana
key = km.getUniqueKey(key);
for (const auto &req : reqs) {
assert(deps.find(req) != deps.end());
seqassert(deps.find(req) != deps.end(), "required key '{}' not found", req);
deps[req].push_back(key);
}

2
codon/sir/util/cloning.cpp
View File

@ -35,7 +35,7 @@ void CloneVisitor::visit(const BodiedFunc *v) {
if (v->getBody())
res->setBody(clone(v->getBody()));
res->setBuiltin(v->isBuiltin());
res->setJIT(v->isJIT());
result = res;
}

2
codon/sir/util/matching.cpp
View File

@ -47,7 +47,7 @@ public:
result = compareFuncs(x, y) &&
std::equal(x->begin(), x->end(), y->begin(), y->end(),
[this](auto *x, auto *y) { return process(x, y); }) &&
process(x->getBody(), y->getBody()) && x->isBuiltin() == y->isBuiltin();
process(x->getBody(), y->getBody()) && x->isJIT() == y->isJIT();
}
VISIT(ExternalFunc);
void handle(const ExternalFunc *x, const ExternalFunc *y) {

61
codon/util/common.cpp
View File

@ -1,24 +1,33 @@
#include "codon/util/common.h"
#include "common.h"
#include <cstdlib>
#include <iostream>
#include <string>
#include <vector>
namespace codon {
namespace {
void compilationMessage(const std::string &header, const std::string &msg,
const std::string &file, int line, int col) {
assert(!(file.empty() && (line > 0 || col > 0)));
assert(!(col > 0 && line <= 0));
std::cerr << "\033[1m";
auto &out = getLogger().err;
seqassert(!(file.empty() && (line > 0 || col > 0)),
"empty filename with non-zero line/col: file={}, line={}, col={}", file,
line, col);
seqassert(!(col > 0 && line <= 0), "col but no line: file={}, line={}, col={}", file,
line, col);
out << "\033[1m";
if (!file.empty())
std::cerr << file.substr(file.rfind('/') + 1);
out << file.substr(file.rfind('/') + 1);
if (line > 0)
std::cerr << ":" << line;
out << ":" << line;
if (col > 0)
std::cerr << ":" << col;
out << ":" << col;
if (!file.empty())
std::cerr << ": ";
std::cerr << header << "\033[1m " << msg << "\033[0m" << std::endl;
out << ": ";
out << header << "\033[1m " << msg << "\033[0m" << std::endl;
}
std::vector<Logger> loggers;
} // namespace
void compilationError(const std::string &msg, const std::string &file, int line,
@ -34,12 +43,36 @@ void compilationWarning(const std::string &msg, const std::string &file, int lin
if (terminate)
exit(EXIT_FAILURE);
}
void Logger::parse(const std::string &s) {
flags |= s.find('t') != std::string::npos ? FLAG_TIME : 0;
flags |= s.find('r') != std::string::npos ? FLAG_REALIZE : 0;
flags |= s.find('T') != std::string::npos ? FLAG_TYPECHECK : 0;
flags |= s.find('i') != std::string::npos ? FLAG_IR : 0;
flags |= s.find('l') != std::string::npos ? FLAG_USER : 0;
}
} // namespace codon
void _seqassert(const char *expr_str, const char *file, int line,
const std::string &msg) {
std::cerr << "Assert failed:\t" << msg << "\n"
<< "Expression:\t" << expr_str << "\n"
<< "Source:\t\t" << file << ":" << line << "\n";
codon::Logger &codon::getLogger() {
if (loggers.empty())
loggers.emplace_back();
return loggers.back();
}
void codon::pushLogger() { loggers.emplace_back(); }
bool codon::popLogger() {
if (loggers.empty())
return false;
loggers.pop_back();
return true;
}
void codon::assertionFailure(const char *expr_str, const char *file, int line,
const std::string &msg) {
auto &out = getLogger().err;
out << "Assert failed:\t" << msg << "\n"
<< "Expression:\t" << expr_str << "\n"
<< "Source:\t\t" << file << ":" << line << "\n";
abort();
}

109
codon/util/common.h
View File

@ -1,13 +1,9 @@
#pragma once
#include <cassert>
#include <climits>
#include <cstdint>
#include <libgen.h>
#include <memory>
#include <chrono>
#include <filesystem>
#include <iostream>
#include <ostream>
#include <stdexcept>
#include <sys/stat.h>
#include "codon/config/config.h"
#include "codon/util/fmt/format.h"
@ -16,68 +12,122 @@
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
extern int _dbg_level;
extern int _level;
#define DBG(c, ...) fmt::print("{}" c "\n", std::string(2 * _level, ' '), ##__VA_ARGS__)
#define DBG(c, ...) \
fmt::print(codon::getLogger().log, "{}" c "\n", \
std::string(2 * codon::getLogger().level, ' '), ##__VA_ARGS__)
#define LOG(c, ...) DBG(c, ##__VA_ARGS__)
#define LOG_TIME(c, ...) \
{ \
if (_dbg_level & (1 << 0)) \
if (codon::getLogger().flags & codon::Logger::FLAG_TIME) \
DBG(c, ##__VA_ARGS__); \
}
#define LOG_REALIZE(c, ...) \
{ \
if (_dbg_level & (1 << 2)) \
if (codon::getLogger().flags & codon::Logger::FLAG_REALIZE) \
DBG(c, ##__VA_ARGS__); \
}
#define LOG_TYPECHECK(c, ...) \
{ \
if (_dbg_level & (1 << 4)) \
if (codon::getLogger().flags & codon::Logger::FLAG_TYPECHECK) \
DBG(c, ##__VA_ARGS__); \
}
#define LOG_IR(c, ...) \
{ \
if (_dbg_level & (1 << 6)) \
if (codon::getLogger().flags & codon::Logger::FLAG_IR) \
DBG(c, ##__VA_ARGS__); \
}
#define LOG_USER(c, ...) \
{ \
if (_dbg_level & (1 << 7)) \
if (codon::getLogger().flags & codon::Logger::FLAG_USER) \
DBG(c, ##__VA_ARGS__); \
}
#define CAST(s, T) dynamic_cast<T *>(s.get())
#define TIME(name) codon::Timer __timer(name)
#ifndef NDEBUG
#define seqassert(expr, msg, ...) \
((expr) ? (void)(0) \
: _seqassert(#expr, __FILE__, __LINE__, fmt::format(msg, ##__VA_ARGS__)))
: codon::assertionFailure(#expr, __FILE__, __LINE__, \
fmt::format(msg, ##__VA_ARGS__)))
#else
#define seqassert(expr, msg, ...) ;
#endif
#pragma clang diagnostic pop
void _seqassert(const char *expr_str, const char *file, int line,
const std::string &msg);
namespace codon {
void assertionFailure(const char *expr_str, const char *file, int line,
const std::string &msg);
struct Logger {
static constexpr int FLAG_TIME = (1 << 0);
static constexpr int FLAG_REALIZE = (1 << 1);
static constexpr int FLAG_TYPECHECK = (1 << 2);
static constexpr int FLAG_IR = (1 << 3);
static constexpr int FLAG_USER = (1 << 4);
int flags;
int level;
std::ostream &out;
std::ostream &err;
std::ostream &log;
Logger() : flags(0), level(0), out(std::cout), err(std::cerr), log(std::clog) {}
void parse(const std::string &logs);
};
Logger &getLogger();
void pushLogger();
bool popLogger();
class Timer {
private:
using clock_type = std::chrono::high_resolution_clock;
std::string name;
std::chrono::time_point<clock_type> start, end;
bool logged;
public:
void log() {
if (!logged) {
end = clock_type::now();
auto elapsed =
std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count() /
1000.0;
LOG_TIME("[T] {} = {:.1f}", name, elapsed);
logged = true;
}
}
Timer(std::string name) : name(std::move(name)), start(), end(), logged(false) {
start = clock_type::now();
}
~Timer() { log(); }
};
struct SrcInfo {
std::string file;
int line;
int col;
int len;
int id; /// used to differentiate different
int id; /// used to differentiate different instances
SrcInfo(std::string file, int line, int col, int len)
: file(std::move(file)), line(line), col(col), len(len) {
static int _id(0);
id = _id++;
: file(std::move(file)), line(line), col(col), len(len), id(0) {
static int nextId = 0;
id = nextId++;
};
SrcInfo() : SrcInfo("", 0, 0, 0){};
friend std::ostream &operator<<(std::ostream &out, const codon::SrcInfo &c) {
char buf[PATH_MAX + 1];
strncpy(buf, c.file.c_str(), PATH_MAX);
auto f = basename(buf);
out << f << ":" << c.line << ":" << c.col;
SrcInfo() : SrcInfo("", 0, 0, 0) {}
friend std::ostream &operator<<(std::ostream &out, const codon::SrcInfo &src) {
out << std::filesystem::path(src.file).filename() << ":" << src.line << ":"
<< src.col;
return out;
}
bool operator==(const SrcInfo &src) const { return id == src.id; }
};
@ -101,4 +151,5 @@ void compilationError(const std::string &msg, const std::string &file = "",
void compilationWarning(const std::string &msg, const std::string &file = "",
int line = 0, int col = 0, bool terminate = false);
} // namespace codon

9
extra/jupyter/share/jupyter/kernels/codon/kernel.json.in 100644
View File

@ -0,0 +1,9 @@
{
"display_name": "Codon",
"argv": [
"@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/@EXECUTABLE_NAME@",
"jupyter",
"{connection_file}"
],
"language": "python"
}

102
extra/jupyter/src/codon.cpp 100644
View File

@ -0,0 +1,102 @@
#include "codon.h"
#ifdef CODON_JUPYTER
#include <dirent.h>
#include <fcntl.h>
#include <iostream>
#include <nlohmann/json.hpp>
#include <unistd.h>
#include <xeus/xhelper.hpp>
#include <xeus/xkernel.hpp>
#include <xeus/xkernel_configuration.hpp>
#include <xeus/xserver_zmq.hpp>
#include "codon/compiler/compiler.h"
#include "codon/compiler/error.h"
#include "codon/compiler/jit.h"
#include "codon/parser/common.h"
#include "codon/util/common.h"
using std::move;
using std::string;
namespace nl = nlohmann;
namespace codon {
CodonJupyter::CodonJupyter(const std::string &argv0) : argv0(argv0) {}
nl::json CodonJupyter::execute_request_impl(int execution_counter, const string &code,
bool silent, bool store_history,
nl::json user_expressions,
bool allow_stdin) {
auto result = jit->exec(code);
string failed;
llvm::handleAllErrors(
result.takeError(),
[&](const codon::error::ParserErrorInfo &e) {
std::vector<string> backtrace;
for (auto &msg : e)
backtrace.push_back(msg.getMessage());
string err = backtrace[0];
backtrace.erase(backtrace.begin());
failed = fmt::format("Compile error: {}\nBacktrace:\n{}", err,
ast::join(backtrace, " \n"));
},
[&](const codon::error::RuntimeErrorInfo &e) {
auto backtrace = e.getBacktrace();
failed = fmt::format("Runtime error: {}\nBacktrace:\n{}", e.getMessage(),
ast::join(backtrace, " \n"));
});
if (failed.empty()) {
nl::json pub_data;
pub_data["text/plain"] = *result;
publish_execution_result(execution_counter, move(pub_data), nl::json::object());
return nl::json{{"status", "ok"},
{"payload", nl::json::array()},
{"user_expressions", nl::json::object()}};
} else {
publish_stream("stderr", failed);
return nl::json{{"status", "error"}};
}
}
void CodonJupyter::configure_impl() {
jit = std::make_unique<codon::jit::JIT>(argv0, "jupyter");
llvm::cantFail(jit->init());
}
nl::json CodonJupyter::complete_request_impl(const string &code, int cursor_pos) {
return nl::json{{"status", "ok"}};
}
nl::json CodonJupyter::inspect_request_impl(const string &code, int cursor_pos,
int detail_level) {
return nl::json{{"status", "ok"}};
}
nl::json CodonJupyter::is_complete_request_impl(const string &code) {
return nl::json{{"status", "complete"}};
}
nl::json CodonJupyter::kernel_info_request_impl() {
return xeus::create_info_reply("1.0", "codon_kernel", "0.1.0", "python", "3.7",
"text/x-python", ".seq", "python", "", "",
"Codon Kernel");
}
void CodonJupyter::shutdown_request_impl() {}
int startJupyterKernel(const std::string &argv0, const std::string &configPath) {
xeus::xconfiguration config = xeus::load_configuration(configPath);
auto context = xeus::make_context<zmq::context_t>();
auto interpreter = std::make_unique<CodonJupyter>(argv0);
xeus::xkernel kernel(config, xeus::get_user_name(), move(context), move(interpreter),
xeus::make_xserver_zmq);
kernel.start();
return 0;
}
} // namespace codon
#endif

40
extra/jupyter/src/codon.h 100644
View File

@ -0,0 +1,40 @@
#pragma once
#ifdef CODON_JUPYTER
#include <codon/compiler/jit.h>
#include <nlohmann/json.hpp>
#include <xeus/xinterpreter.hpp>
using xeus::xinterpreter;
namespace nl = nlohmann;
namespace codon {
class CodonJupyter : public xinterpreter {
std::unique_ptr<codon::jit::JIT> jit;
std::string argv0;
public:
CodonJupyter(const std::string &argv0);
private:
void configure_impl() override;
nl::json execute_request_impl(int execution_counter, const std::string &code,
bool silent, bool store_history,
nl::json user_expressions, bool allow_stdin) override;
nl::json complete_request_impl(const std::string &code, int cursor_pos) override;
nl::json inspect_request_impl(const std::string &code, int cursor_pos,
int detail_level) override;
nl::json is_complete_request_impl(const std::string &code) override;
nl::json kernel_info_request_impl() override;
void shutdown_request_impl() override;
};
int startJupyterKernel(const std::string &argv0, const std::string &configPath);
} // namespace codon
#endif

1
stdlib/internal/__init__.codon
View File

@ -25,6 +25,7 @@ from internal.types.collections.dict import *
import internal.c_stubs as _C
from internal.builtin import *
from internal.builtin import _jit_display
from internal.box import Box
from internal.str import *

11
stdlib/internal/builtin.codon
View File

@ -329,3 +329,14 @@ class int:
raise ValueError("invalid literal for int() with base " + str(base) + ": " + s)
return result
def _jit_display(x, s: Static[str]):
if hasattr(x, "__repr_pretty__") and s == "jupyter":
return x.__repr_pretty__()
elif hasattr(x, "__repr__"):
return x.__repr__()
elif hasattr(x, "__str__"):
return x.__str__()
else:
return ''

9
stdlib/internal/dlopen.codon
View File

@ -9,6 +9,7 @@ from C import dlsym(cobj, cobj) -> cobj as c_dlsym
from C import dlclose(cobj) -> i32 as c_dlclose
RTLD_NOW = 2
RTLD_GLOBAL = 8 if seq_is_macos() else 256
RTLD_LOCAL = 0 if seq_is_macos() else 256
def dlext():
if seq_is_macos():
@ -26,8 +27,12 @@ def dlopen(name: str, flag: int = RTLD_NOW | RTLD_GLOBAL) -> cobj:
raise CError(dlerror())
return h
def dlsym[Fn](lib: str, name: str) -> Fn:
h = dlopen(lib)
def dlsym[Fn](lib, name: str) -> Fn:
h = cobj()
if isinstance(lib, str):
h = dlopen(lib)
else:
h = lib
fn = c_dlsym(h, name.c_str())
if fn == cobj():
raise CError(dlerror())

71
stdlib/internal/python.codon
View File

@ -1,5 +1,5 @@
import os
from internal.dlopen import dlext
from internal.dlopen import *
PyUnicode_AsEncodedString = Function[[cobj, cobj, cobj], cobj](cobj())
PyBytes_AsString = Function[[cobj], cobj](cobj())
@ -45,75 +45,76 @@ def init():
return
LD = os.getenv('CODON_PYTHON', default='libpython.' + dlext())
hnd = dlopen(LD, RTLD_LOCAL | RTLD_NOW)
global PyUnicode_AsEncodedString
PyUnicode_AsEncodedString = _dlsym(LD, "PyUnicode_AsEncodedString")
PyUnicode_AsEncodedString = dlsym(hnd, "PyUnicode_AsEncodedString")
global PyBytes_AsString
PyBytes_AsString = _dlsym(LD, "PyBytes_AsString")
PyBytes_AsString = dlsym(hnd, "PyBytes_AsString")
global PyErr_Fetch
PyErr_Fetch = _dlsym(LD, "PyErr_Fetch")
PyErr_Fetch = dlsym(hnd, "PyErr_Fetch")
global PyObject_GetAttrString
PyObject_GetAttrString = _dlsym(LD, "PyObject_GetAttrString")
PyObject_GetAttrString = dlsym(hnd, "PyObject_GetAttrString")
global PyObject_GetAttr
PyObject_GetAttr = _dlsym(LD, "PyObject_GetAttr")
PyObject_GetAttr = dlsym(hnd, "PyObject_GetAttr")
global PyObject_Str
PyObject_Str = _dlsym(LD, "PyObject_Str")
PyObject_Str = dlsym(hnd, "PyObject_Str")
global PyRun_SimpleString
PyRun_SimpleString = _dlsym(LD, "PyRun_SimpleString")
PyRun_SimpleString = dlsym(hnd, "PyRun_SimpleString")
global Py_IncRef
Py_IncRef = _dlsym(LD, "Py_IncRef")
Py_IncRef = dlsym(hnd, "Py_IncRef")
global Py_DecRef
Py_DecRef = _dlsym(LD, "Py_DecRef")
Py_DecRef = dlsym(hnd, "Py_DecRef")
global PyObject_Call
PyObject_Call = _dlsym(LD, "PyObject_Call")
PyObject_Call = dlsym(hnd, "PyObject_Call")
global PyObject_SetAttrString
PyObject_SetAttrString = _dlsym(LD, "PyObject_SetAttrString")
PyObject_SetAttrString = dlsym(hnd, "PyObject_SetAttrString")
global PyObject_Length
PyObject_Length = _dlsym(LD, "PyObject_Length")
PyObject_Length = dlsym(hnd, "PyObject_Length")
global Py_Initialize
Py_Initialize = _dlsym(LD, "Py_Initialize")
Py_Initialize = dlsym(hnd, "Py_Initialize")
global PyImport_ImportModule
PyImport_ImportModule = _dlsym(LD, "PyImport_ImportModule")
PyImport_ImportModule = dlsym(hnd, "PyImport_ImportModule")
global PyLong_FromLong
PyLong_FromLong = _dlsym(LD, "PyLong_FromLong")
PyLong_FromLong = dlsym(hnd, "PyLong_FromLong")
global PyLong_AsLong
PyLong_AsLong = _dlsym(LD, "PyLong_AsLong")
PyLong_AsLong = dlsym(hnd, "PyLong_AsLong")
global PyFloat_FromDouble
PyFloat_FromDouble = _dlsym(LD, "PyFloat_FromDouble")
PyFloat_FromDouble = dlsym(hnd, "PyFloat_FromDouble")
global PyFloat_AsDouble
PyFloat_AsDouble = _dlsym(LD, "PyFloat_AsDouble")
PyFloat_AsDouble = dlsym(hnd, "PyFloat_AsDouble")
global PyBool_FromLong
PyBool_FromLong = _dlsym(LD, "PyBool_FromLong")
PyBool_FromLong = dlsym(hnd, "PyBool_FromLong")
global PyObject_IsTrue
PyObject_IsTrue = _dlsym(LD, "PyObject_IsTrue")
PyObject_IsTrue = dlsym(hnd, "PyObject_IsTrue")
global PyUnicode_DecodeFSDefaultAndSize
PyUnicode_DecodeFSDefaultAndSize = _dlsym(LD, "PyUnicode_DecodeFSDefaultAndSize")
PyUnicode_DecodeFSDefaultAndSize = dlsym(hnd, "PyUnicode_DecodeFSDefaultAndSize")
global PyTuple_New
PyTuple_New = _dlsym(LD, "PyTuple_New")
PyTuple_New = dlsym(hnd, "PyTuple_New")
global PyTuple_SetItem
PyTuple_SetItem = _dlsym(LD, "PyTuple_SetItem")
PyTuple_SetItem = dlsym(hnd, "PyTuple_SetItem")
global PyTuple_GetItem
PyTuple_GetItem = _dlsym(LD, "PyTuple_GetItem")
PyTuple_GetItem = dlsym(hnd, "PyTuple_GetItem")
global PyList_New
PyList_New = _dlsym(LD, "PyList_New")
PyList_New = dlsym(hnd, "PyList_New")
global PyList_SetItem
PyList_SetItem = _dlsym(LD, "PyList_SetItem")
PyList_SetItem = dlsym(hnd, "PyList_SetItem")
global PyList_GetItem
PyList_GetItem = _dlsym(LD, "PyList_GetItem")
PyList_GetItem = dlsym(hnd, "PyList_GetItem")
global PySet_New
PySet_New = _dlsym(LD, "PySet_New")
PySet_New = dlsym(hnd, "PySet_New")
global PySet_Add
PySet_Add = _dlsym(LD, "PySet_Add")
PySet_Add = dlsym(hnd, "PySet_Add")
global PyDict_New
PyDict_New = _dlsym(LD, "PyDict_New")
PyDict_New = dlsym(hnd, "PyDict_New")
global PyDict_SetItem
PyDict_SetItem = _dlsym(LD, "PyDict_SetItem")
PyDict_SetItem = dlsym(hnd, "PyDict_SetItem")
global PyDict_Next
PyDict_Next = _dlsym(LD, "PyDict_Next")
PyDict_Next = dlsym(hnd, "PyDict_Next")
global PyObject_GetIter
PyObject_GetIter = _dlsym(LD, "PyObject_GetIter")
PyObject_GetIter = dlsym(hnd, "PyObject_GetIter")
global PyIter_Next
PyIter_Next = _dlsym(LD, "PyIter_Next")
PyIter_Next = dlsym(hnd, "PyIter_Next")
Py_Initialize()
_PY_INITIALIZED = True

41
test/main.cpp
View File

@ -1,4 +1,5 @@
#include <algorithm>
#include <cstdio>
#include <dirent.h>
#include <fcntl.h>
#include <fstream>
@ -12,15 +13,14 @@
#include <unistd.h>
#include <vector>
#include "codon/compiler/compiler.h"
#include "codon/compiler/error.h"
#include "codon/parser/common.h"
#include "codon/parser/parser.h"
#include "codon/sir/llvm/llvisitor.h"
#include "codon/sir/transform/manager.h"
#include "codon/sir/transform/pass.h"
#include "codon/sir/util/inlining.h"
#include "codon/sir/util/irtools.h"
#include "codon/sir/util/outlining.h"
#include "codon/util/common.h"
#include "gtest/gtest.h"
using namespace codon;
@ -178,22 +178,33 @@ public:
close(out_pipe[1]);
auto file = getFilename(get<0>(GetParam()));
bool debug = get<1>(GetParam());
auto code = get<3>(GetParam());
auto startLine = get<4>(GetParam());
auto *module = parse(argv0, file, code, !code.empty(),
/* isTest */ 1 + get<5>(GetParam()), startLine);
if (!module)
exit(EXIT_FAILURE);
int testFlags = 1 + get<5>(GetParam());
ir::transform::PassManager pm;
pm.registerPass(std::make_unique<TestOutliner>());
pm.registerPass(std::make_unique<TestInliner>());
pm.run(module);
auto compiler = std::make_unique<Compiler>(
argv0, debug, /*disabledPasses=*/std::vector<std::string>{}, /*isTest=*/true);
compiler->getLLVMVisitor()->setStandalone(
true); // make sure we abort() on runtime error
llvm::handleAllErrors(code.empty()
? compiler->parseFile(file, testFlags)
: compiler->parseCode(file, code, startLine, testFlags),
[](const error::ParserErrorInfo &e) {
for (auto &msg : e) {
getLogger().level = 0;
printf("%s\n", msg.getMessage().c_str());
}
fflush(stdout);
exit(EXIT_FAILURE);
});
ir::LLVMVisitor visitor(/*debug=*/get<1>(GetParam()));
visitor.visit(module);
visitor.run({file});
auto *pm = compiler->getPassManager();
pm->registerPass(std::make_unique<TestOutliner>());
pm->registerPass(std::make_unique<TestInliner>());
llvm::cantFail(compiler->compile());
compiler->getLLVMVisitor()->run({file});
fflush(stdout);
exit(EXIT_SUCCESS);
} else {

6
test/sir/func.cpp
View File

@ -32,8 +32,8 @@ TEST_F(SIRCoreTest, FuncRealizationAndVarInsertionEraseAndIterators) {
TEST_F(SIRCoreTest, BodiedFuncQueryAndReplace) {
auto *fn = module->Nr<BodiedFunc>();
fn->realize(module->unsafeGetDummyFuncType(), {});
fn->setBuiltin();
ASSERT_TRUE(fn->isBuiltin());
fn->setJIT();
ASSERT_TRUE(fn->isJIT());
auto *body = fn->getBody();
ASSERT_FALSE(body);
@ -63,7 +63,7 @@ TEST_F(SIRCoreTest, BodiedFuncCloning) {
auto *fn = module->Nr<BodiedFunc>("fn");
fn->realize(module->unsafeGetDummyFuncType(), {});
fn->setBuiltin();
fn->setJIT();
fn->setBody(module->Nr<SeriesFlow>());
ASSERT_TRUE(util::match(fn, cv->clone(fn)));
}

6
test/sir/util/matching.cpp
View File

@ -25,8 +25,8 @@ TEST_F(SIRCoreTest, MatchingEquivalentFunc) {
auto *second = module->Nr<BodiedFunc>();
second->realize(module->unsafeGetDummyFuncType(), {});
first->setBuiltin();
second->setBuiltin();
first->setJIT();
second->setJIT();
ASSERT_TRUE(util::match(first, second));
}
@ -58,7 +58,7 @@ TEST_F(SIRCoreTest, MatchingNonEquivalentFunc) {
auto *second = module->Nr<BodiedFunc>();
second->realize(module->unsafeGetDummyFuncType(), {});
first->setBuiltin();
first->setJIT();
ASSERT_FALSE(util::match(first, second));
}

1
test/types.cpp
View File

@ -13,7 +13,6 @@
#include <vector>
#include "codon/parser/common.h"
#include "codon/parser/parser.h"
#include "codon/util/common.h"
#include "gtest/gtest.h"