codon build command: add --cir output type option (#649)

CMakeLists.txt

@@ -482,7 +482,7 @@ target_link_libraries(codon PUBLIC fmt codonc codon_jupyter Threads::Threads)
 include(FetchContent)
 FetchContent_Declare(
   googletest
-  URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
+  URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip
 )
 # For Windows: Prevent overriding the parent project's compiler/linker settings
 set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)

README.md

@@ -149,7 +149,7 @@ print(total)
 ```
 
 Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic
-reduction to avoid race conditions. Learn more in the [multitheading docs](advanced/parallel.md).
+reduction to avoid race conditions. Learn more in the [multithreading docs](https://docs.exaloop.io/codon/advanced/parallel).
 
 Codon also supports writing and executing GPU kernels. Here's an example that computes the
 [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set):

cmake/deps.cmake

@@ -1,8 +1,8 @@
-set(CPM_DOWNLOAD_VERSION 0.32.3)
+set(CPM_DOWNLOAD_VERSION 0.40.8)
 set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
 if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION}))
     message(STATUS "Downloading CPM.cmake...")
-    file(DOWNLOAD https://github.com/TheLartians/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake ${CPM_DOWNLOAD_LOCATION})
+    file(DOWNLOAD https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake ${CPM_DOWNLOAD_LOCATION})
 endif()
 include(${CPM_DOWNLOAD_LOCATION})
 
@@ -77,9 +77,9 @@ endif()
 
 CPMAddPackage(
     NAME bdwgc
-    GITHUB_REPOSITORY "ivmai/bdwgc"
+    GITHUB_REPOSITORY "exaloop/bdwgc"
     VERSION 8.0.5
-    GIT_TAG d0ba209660ea8c663e06d9a68332ba5f42da54ba
+    GIT_TAG e16c67244aff26802203060422545d38305e0160
     EXCLUDE_FROM_ALL YES
     OPTIONS "CMAKE_POSITION_INDEPENDENT_CODE ON"
             "BUILD_SHARED_LIBS OFF"
@@ -169,7 +169,7 @@ if(NOT APPLE)
     CPMAddPackage(
         NAME openblas
         GITHUB_REPOSITORY "OpenMathLib/OpenBLAS"
-        GIT_TAG v0.3.28
+        GIT_TAG v0.3.29
         EXCLUDE_FROM_ALL YES
         OPTIONS "DYNAMIC_ARCH ON"
                 "BUILD_TESTING OFF"

codon/app/main.cpp

@@ -11,6 +11,7 @@
 #include <unordered_map>
 #include <vector>
 
+#include "codon/cir/util/format.h"
 #include "codon/compiler/compiler.h"
 #include "codon/compiler/error.h"
 #include "codon/compiler/jit.h"
@@ -87,7 +88,7 @@ void initLogFlags(const llvm::cl::opt<std::string> &log) {
     codon::getLogger().parse(std::string(d));
 }
 
-enum BuildKind { LLVM, Bitcode, Object, Executable, Library, PyExtension, Detect };
+enum BuildKind { LLVM, Bitcode, Object, Executable, Library, PyExtension, Detect, CIR };
 enum OptMode { Debug, Release };
 enum Numerics { C, Python };
 } // namespace
@@ -333,6 +334,7 @@ int buildMode(const std::vector<const char *> &args, const std::string &argv0) {
           clEnumValN(Executable, "exe", "Generate executable"),
           clEnumValN(Library, "lib", "Generate shared library"),
           clEnumValN(PyExtension, "pyext", "Generate Python extension module"),
+          clEnumValN(CIR, "cir", "Generate Codon Intermediate Representation"),
           clEnumValN(Detect, "detect",
                      "Detect output type based on output file extension")),
       llvm::cl::init(Detect));
@@ -372,6 +374,9 @@ int buildMode(const std::vector<const char *> &args, const std::string &argv0) {
   case BuildKind::Detect:
     extension = "";
     break;
+  case BuildKind::CIR:
+    extension = ".cir";
+    break;
   default:
     seqassertn(0, "unknown build kind");
   }
@@ -401,6 +406,11 @@ int buildMode(const std::vector<const char *> &args, const std::string &argv0) {
     compiler->getLLVMVisitor()->writeToPythonExtension(*compiler->getCache()->pyModule,
                                                        filename);
     break;
+  case BuildKind::CIR: {
+    std::ofstream out(filename);
+    codon::ir::util::format(out, compiler->getModule());
+    break;
+  }
   case BuildKind::Detect:
     compiler->getLLVMVisitor()->compile(filename, argv0, libsVec, lflags);
     break;

docs/README.md

@@ -124,7 +124,7 @@ print(total)
 ```
 
 Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic
-reduction to avoid race conditions. Learn more in the [multitheading docs](advanced/parallel.md).
+reduction to avoid race conditions. Learn more in the [multithreading docs](advanced/parallel.md).
 
 Codon also supports writing and executing GPU kernels. Here's an example that computes the
 [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set):

stdlib/internal/python.codon

@@ -24,6 +24,7 @@ PyFloat_AsDouble = Function[[cobj], float](cobj())
 PyFloat_FromDouble = Function[[float], cobj](cobj())
 PyBool_FromLong = Function[[int], cobj](cobj())
 PyBytes_AsString = Function[[cobj], cobj](cobj())
+PyBytes_Size = Function[[cobj], int](cobj())
 PyList_New = Function[[int], cobj](cobj())
 PyList_Size = Function[[cobj], int](cobj())
 PyList_GetItem = Function[[cobj, int], cobj](cobj())
@@ -130,6 +131,7 @@ PyLong_Type = cobj()
 PyFloat_Type = cobj()
 PyBool_Type = cobj()
 PyUnicode_Type = cobj()
+PyBytes_Type = cobj()
 PyComplex_Type = cobj()
 PyList_Type = cobj()
 PyDict_Type = cobj()
@@ -213,6 +215,7 @@ def init_handles_dlopen(py_handle: cobj):
     global PyFloat_FromDouble
     global PyBool_FromLong
     global PyBytes_AsString
+    global PyBytes_Size
     global PyList_New
     global PyList_Size
     global PyList_GetItem
@@ -303,6 +306,7 @@ def init_handles_dlopen(py_handle: cobj):
     global PyFloat_Type
     global PyBool_Type
     global PyUnicode_Type
+    global PyBytes_Type
     global PyComplex_Type
     global PyList_Type
     global PyDict_Type
@@ -347,6 +351,7 @@ def init_handles_dlopen(py_handle: cobj):
     PyFloat_FromDouble = dlsym(py_handle, "PyFloat_FromDouble")
     PyBool_FromLong = dlsym(py_handle, "PyBool_FromLong")
     PyBytes_AsString = dlsym(py_handle, "PyBytes_AsString")
+    PyBytes_Size = dlsym(py_handle, "PyBytes_Size")
     PyList_New = dlsym(py_handle, "PyList_New")
     PyList_Size = dlsym(py_handle, "PyList_Size")
     PyList_GetItem = dlsym(py_handle, "PyList_GetItem")
@@ -437,6 +442,7 @@ def init_handles_dlopen(py_handle: cobj):
     PyFloat_Type = dlsym(py_handle, "PyFloat_Type")
     PyBool_Type = dlsym(py_handle, "PyBool_Type")
     PyUnicode_Type = dlsym(py_handle, "PyUnicode_Type")
+    PyBytes_Type = dlsym(py_handle, "PyBytes_Type")
     PyComplex_Type = dlsym(py_handle, "PyComplex_Type")
     PyList_Type = dlsym(py_handle, "PyList_Type")
     PyDict_Type = dlsym(py_handle, "PyDict_Type")
@@ -482,6 +488,7 @@ def init_handles_static():
     from C import PyFloat_FromDouble(float) -> cobj as _PyFloat_FromDouble
     from C import PyBool_FromLong(int) -> cobj as _PyBool_FromLong
     from C import PyBytes_AsString(cobj) -> cobj as _PyBytes_AsString
+    from C import PyBytes_Size(cobj) -> int as _PyBytes_Size
     from C import PyList_New(int) -> cobj as _PyList_New
     from C import PyList_Size(cobj) -> int as _PyList_Size
     from C import PyList_GetItem(cobj, int) -> cobj as _PyList_GetItem
@@ -572,6 +579,7 @@ def init_handles_static():
     from C import PyFloat_Type: cobj as _PyFloat_Type
     from C import PyBool_Type: cobj as _PyBool_Type
     from C import PyUnicode_Type: cobj as _PyUnicode_Type
+    from C import PyBytes_Type: cobj as _PyBytes_Type
     from C import PyComplex_Type: cobj as _PyComplex_Type
     from C import PyList_Type: cobj as _PyList_Type
     from C import PyDict_Type: cobj as _PyDict_Type
@@ -616,6 +624,7 @@ def init_handles_static():
     global PyFloat_FromDouble
     global PyBool_FromLong
     global PyBytes_AsString
+    global PyBytes_Size
     global PyList_New
     global PyList_Size
     global PyList_GetItem
@@ -706,6 +715,7 @@ def init_handles_static():
     global PyFloat_Type
     global PyBool_Type
     global PyUnicode_Type
+    global PyBytes_Type
     global PyComplex_Type
     global PyList_Type
     global PyDict_Type
@@ -750,6 +760,7 @@ def init_handles_static():
     PyFloat_FromDouble = _PyFloat_FromDouble
     PyBool_FromLong = _PyBool_FromLong
     PyBytes_AsString = _PyBytes_AsString
+    PyBytes_Size = _PyBytes_Size
     PyList_New = _PyList_New
     PyList_Size = _PyList_Size
     PyList_GetItem = _PyList_GetItem
@@ -840,6 +851,7 @@ def init_handles_static():
     PyFloat_Type = __ptr__(_PyFloat_Type).as_byte()
     PyBool_Type = __ptr__(_PyBool_Type).as_byte()
     PyUnicode_Type = __ptr__(_PyUnicode_Type).as_byte()
+    PyBytes_Type = __ptr__(_PyBytes_Type).as_byte()
     PyComplex_Type = __ptr__(_PyComplex_Type).as_byte()
     PyList_Type = __ptr__(_PyList_Type).as_byte()
     PyDict_Type = __ptr__(_PyDict_Type).as_byte()
@@ -1174,7 +1186,7 @@ class pyobj:
         return pyobj.to_str(self.p, errors, empty)
 
     def to_str(p: cobj, errors: str, empty: str = "") -> str:
-        obj = PyUnicode_AsEncodedString(p, "utf-8".c_str(), errors.c_str())
+        obj = PyUnicode_AsEncodedString(p, "utf-8".ptr, errors.c_str() if errors else "".ptr)
         if obj == cobj():
             return empty
         bts = PyBytes_AsString(obj)
@@ -1292,8 +1304,11 @@ class _PyObject_Struct:
 def _conversion_error(name: Static[str]):
     raise PyError("conversion error: Python object did not have type '" + name + "'")
 
+def _get_type(o: cobj):
+    return Ptr[_PyObject_Struct](o)[0].pytype
+
 def _ensure_type(o: cobj, t: cobj, name: Static[str]):
-    if Ptr[_PyObject_Struct](o)[0].pytype != t:
+    if _get_type(o) != t:
         _conversion_error(name)
 
 
@@ -1350,7 +1365,14 @@ class str:
         return pyobj.exc_wrap(PyUnicode_DecodeFSDefaultAndSize(self.ptr, self.len))
 
     def __from_py__(s: cobj) -> str:
-        return pyobj.exc_wrap(pyobj.to_str(s, "strict"))
+        if _get_type(s) == PyBytes_Type:
+            n = PyBytes_Size(s)
+            p0 = PyBytes_AsString(s)
+            p1 = cobj(n)
+            str.memcpy(p1, p0, n)
+            return str(p1, n)
+        else:
+            return pyobj.exc_wrap(pyobj.to_str(s, "strict"))
 
 @extend
 class complex:

test/python/pyext.py

@@ -88,6 +88,7 @@ def test_codon_extensions(m):
         assert m.f4(a=2.2) == (2.2, 2.22)
         assert m.f4(b=3.3) == (1.11, 3.3)
         assert m.f4('foo') == ('foo', 'foo')
+        assert m.f4(b'foo') == ('foo', 'foo')
         assert m.f4({1}) == {1}
         assert m.f5() is None
         assert equal(m.f6(1.9, 't'), 1.9, 1.9, 't')