1
0
mirror of https://github.com/exaloop/codon.git synced 2025-06-03 15:03:52 +08:00
codon/stdlib/pickle.codon

273 lines
7.5 KiB
Python
Raw Normal View History

Dynamic Polymorphism (#58) * Use Static[] for static inheritance * Support .seq extension * Fix #36 * Polymorphic typechecking; vtables [wip] * v-table dispatch [wip] * vtable routing [wip; bug] * vtable routing [MVP] * Fix texts * Add union type support * Update FAQs * Clarify * Add BSL license * Add makeUnion * Add IR UnionType * Update union representation in LLVM * Update README * Update README.md * Update README * Update README.md * Add benchmarks * Add more benchmarks and README * Add primes benchmark * Update benchmarks * Fix cpp * Clean up list * Update faq.md * Add binary trees benchmark * Add fannkuch benchmark * Fix paths * Add PyPy * Abort on fail * More benchmarks * Add cpp word_count * Update set_partition cpp * Add nbody cpp * Add TAQ cpp; fix word_count timing * Update CODEOWNERS * Update README * Update README.md * Update CODEOWNERS * Fix bench script * Update binary_trees.cpp * Update taq.cpp * Fix primes benchmark * Add mandelbrot benchmark * Fix OpenMP init * Add Module::unsafeGetUnionType * UnionType [wip] [skip ci] * Integrate IR unions and Union * UnionType refactor [skip ci] * Update README.md * Update docs * UnionType [wip] [skip ci] * UnionType and automatic unions * Add Slack * Update faq.md * Refactor types * New error reporting [wip] * New error reporting [wip] * peglib updates [wip] [skip_ci] * Fix parsing issues * Fix parsing issues * Fix error reporting issues * Make sure random module matches Python * Update releases.md * Fix tests * Fix #59 * Fix #57 * Fix #50 * Fix #49 * Fix #26; Fix #51; Fix #47; Fix #49 * Fix collection extension methods * Fix #62 * Handle *args/**kwargs with Callable[]; Fix #43 * Fix #43 * Fix Ptr.__sub__; Fix polymorphism issues * Add typeinfo * clang-format * Upgrade fmtlib to v9; Use CPM for fmtlib; format spec support; __format__ support * Use CPM for semver and toml++ * Remove extension check * Revamp str methods * Update str.zfill * Fix thunk crashes [wip] [skip_ci] * Fix str.__reversed__ * Fix count_with_max * Fix vtable memory allocation issues * Add poly AST tests * Use PDQsort when stability does not matter * Fix dotted imports; Fix issues * Fix kwargs passing to Python * Fix #61 * Fix #37 * Add isinstance support for unions; Union methods return Union type if different * clang-format * Nicely format error tracebacks * Fix build issues; clang-format * Fix OpenMP init * Fix OpenMP init * Update README.md * Fix tests * Update license [skip ci] * Update license [ci skip] * Add copyright header to all source files * Fix super(); Fix error recovery in ClassStmt * Clean up whitespace [ci skip] * Use Python 3.9 on CI * Print info in random test * Fix single unions * Update random_test.codon * Fix polymorhic thunk instantiation * Fix random test * Add operator.attrgetter and operator.methodcaller * Add code documentation * Update documentation * Update README.md * Fix tests * Fix random init Co-authored-by: A. R. Shajii <ars@ars.me>
2022-12-04 16:45:21 -08:00
# Copyright (C) 2022 Exaloop Inc. <https://exaloop.io>
2022-01-24 11:06:34 +01:00
2021-09-27 14:02:44 -04:00
from internal.file import _gz_errcheck
from internal.gc import sizeof, atomic
2022-02-16 16:51:16 +01:00
def pickle(x: T, jar: Jar, T: type):
2021-09-27 14:02:44 -04:00
x.__pickle__(jar)
2022-01-24 11:06:34 +01:00
def unpickle(jar: Jar, T: type) -> T:
2021-09-27 14:02:44 -04:00
return T.__unpickle__(jar)
2022-02-16 16:51:16 +01:00
def dump(x: T, f, T: type):
2021-09-27 14:02:44 -04:00
x.__pickle__(f.fp)
2022-01-24 11:06:34 +01:00
def load(f, T: type) -> T:
2021-09-27 14:02:44 -04:00
return T.__unpickle__(f.fp)
2022-02-16 16:51:16 +01:00
def _write_raw(jar: Jar, p: cobj, n: int):
2022-01-24 11:06:34 +01:00
LIMIT = 0x7FFFFFFF
2021-09-27 14:02:44 -04:00
while n > 0:
b = n if n < LIMIT else LIMIT
status = int(_C.gzwrite(jar, p, u32(b)))
if status != b:
_gz_errcheck(jar)
2022-02-16 16:51:16 +01:00
raise IOError(f"pickle error: gzwrite returned {status}")
2021-09-27 14:02:44 -04:00
p += b
n -= b
2022-02-16 16:51:16 +01:00
def _read_raw(jar: Jar, p: cobj, n: int):
2022-01-24 11:06:34 +01:00
LIMIT = 0x7FFFFFFF
2021-09-27 14:02:44 -04:00
while n > 0:
b = n if n < LIMIT else LIMIT
status = int(_C.gzread(jar, p, u32(b)))
if status != b:
_gz_errcheck(jar)
2022-02-16 16:51:16 +01:00
raise IOError(f"pickle error: gzread returned {status}")
2021-09-27 14:02:44 -04:00
p += b
n -= b
2022-02-16 16:51:16 +01:00
def _write(jar: Jar, x: T, T: type):
2021-09-27 14:02:44 -04:00
y = __ptr__(x)
_write_raw(jar, y.as_byte(), sizeof(T))
2022-01-24 11:06:34 +01:00
def _read(jar: Jar, T: type) -> T:
2021-09-27 14:02:44 -04:00
x = T()
y = __ptr__(x)
_read_raw(jar, y.as_byte(), sizeof(T))
return x
# Extend core types to allow pickling
@extend
class int:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
_write(jar, self)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> int:
2021-09-27 14:02:44 -04:00
return _read(jar, int)
@extend
class Int:
def __pickle__(self, jar: Jar):
_write(jar, self)
def __unpickle__(jar: Jar) -> Int[N]:
return _read(jar, Int[N])
@extend
class UInt:
def __pickle__(self, jar: Jar):
_write(jar, self)
def __unpickle__(jar: Jar) -> UInt[N]:
return _read(jar, UInt[N])
2021-09-27 14:02:44 -04:00
@extend
class float:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
_write(jar, self)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> float:
2021-09-27 14:02:44 -04:00
return _read(jar, float)
GPU and other updates (#52) * Add nvptx pass * Fix spaces * Don't change name * Add runtime support * Add init call * Add more runtime functions * Add launch function * Add intrinsics * Fix codegen * Run GPU pass between general opt passes * Set data layout * Create context * Link libdevice * Add function remapping * Fix linkage * Fix libdevice link * Fix linking * Fix personality * Fix linking * Fix linking * Fix linking * Add internalize pass * Add more math conversions * Add more re-mappings * Fix conversions * Fix __str__ * Add decorator attribute for any decorator * Update kernel decorator * Fix kernel decorator * Fix kernel decorator * Fix kernel decorator * Fix kernel decorator * Remove old decorator * Fix pointer calc * Fix fill-in codegen * Fix linkage * Add comment * Update list conversion * Add more conversions * Add dict and set conversions * Add float32 type to IR/LLVM * Add float32 * Add float32 stdlib * Keep required global values in PTX module * Fix PTX module pruning * Fix malloc * Set will-return * Fix name cleanup * Fix access * Fix name cleanup * Fix function renaming * Update dimension API * Fix args * Clean up API * Move GPU transformations to end of opt pipeline * Fix alloc replacements * Fix naming * Target PTX 4.2 * Fix global renaming * Fix early return in static blocks; Add __realized__ function * Format * Add __llvm_name__ for functions * Add vector type to IR * SIMD support [wip] * Update kernel naming * Fix early returns; Fix SIMD calls * Fix kernel naming * Fix IR matcher * Remove module print * Update realloc * Add overloads for 32-bit float math ops * Add gpu.Pointer type for working with raw pointers * Add float32 conversion * Add to_gpu and from_gpu * clang-format * Add f32 reduction support to OpenMP * Fix automatic GPU class conversions * Fix conversion functions * Fix conversions * Rename self * Fix tuple conversion * Fix conversions * Fix conversions * Update PTX filename * Fix filename * Add raw function * Add GPU docs * Allow nested object conversions * Add tests (WIP) * Update SIMD * Add staticrange and statictuple loop support * SIMD updates * Add new Vec constructors * Fix UInt conversion * Fix size-0 allocs * Add more tests * Add matmul test * Rename gpu test file * Add more tests * Add alloc cache * Fix object_to_gpu * Fix frees * Fix str conversion * Fix set conversion * Fix conversions * Fix class conversion * Fix str conversion * Fix byte conversion * Fix list conversion * Fix pointer conversions * Fix conversions * Fix conversions * Update tests * Fix conversions * Fix tuple conversion * Fix tuple conversion * Fix auto conversions * Fix conversion * Fix magics * Update tests * Support GPU in JIT mode * Fix GPU+JIT * Fix kernel filename in JIT mode * Add __static_print__; Add earlyDefines; Various domination bugfixes; SimplifyContext RAII base handling * Fix global static handling * Fix float32 tests * FIx gpu module * Support OpenMP "collapse" option * Add more collapse tests * Capture generics and statics * TraitVar handling * Python exceptions / isinstance [wip; no_ci] * clang-format * Add list comparison operators * Support empty raise in IR * Add dict 'or' operator * Fix repr * Add copy module * Fix spacing * Use sm_30 * Python exceptions * TypeTrait support; Fix defaultDict * Fix earlyDefines * Add defaultdict * clang-format * Fix invalid canonicalizations * Fix empty raise * Fix copyright * Add Python numerics option * Support py-numerics in math module * Update docs * Add static Python division / modulus * Add static py numerics tests * Fix staticrange/tuple; Add KwTuple.__getitem__ * clang-format * Add gpu parameter to par * Fix globals * Don't init loop vars on loop collapse * Add par-gpu tests * Update gpu docs * Fix isinstance check * Remove invalid test * Add -libdevice to set custom path [skip ci] * Add release notes; bump version [skip ci] * Add libdevice docs [skip ci] Co-authored-by: Ibrahim Numanagić <ibrahimpasa@gmail.com>
2022-09-15 15:40:00 -04:00
@extend
class float32:
def __pickle__(self, jar: Jar):
_write(jar, self)
def __unpickle__(jar: Jar) -> float32:
return _read(jar, float32)
2021-09-27 14:02:44 -04:00
@extend
class bool:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
_write(jar, self)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> bool:
2021-09-27 14:02:44 -04:00
return _read(jar, bool)
@extend
class byte:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
_write(jar, self)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> byte:
2021-09-27 14:02:44 -04:00
return _read(jar, byte)
@extend
class str:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
_write(jar, self.len)
_write_raw(jar, self.ptr, self.len)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> str:
2021-09-27 14:02:44 -04:00
n = _read(jar, int)
p = Ptr[byte](n)
_read_raw(jar, p, n)
return str(p, n)
@extend
class List:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
n = len(self)
pickle(n, jar)
if atomic(T):
_write_raw(jar, (self.arr.ptr).as_byte(), n * sizeof(T))
else:
for i in range(n):
pickle(self.arr[i], jar)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> List[T]:
2021-09-27 14:02:44 -04:00
n = unpickle(jar, int)
arr = Array[T](n)
if atomic(T):
_read_raw(jar, (arr.ptr).as_byte(), n * sizeof(T))
else:
for i in range(n):
arr[i] = unpickle(jar, T)
return List[T](arr, n)
@extend
class DynamicTuple:
def __pickle__(self, jar: Jar):
n = len(self)
pickle(n, jar)
if atomic(T):
_write_raw(jar, (self._ptr).as_byte(), n * sizeof(T))
else:
for i in range(n):
pickle(self._ptr[i], jar)
def __unpickle__(jar: Jar) -> DynamicTuple[T]:
n = unpickle(jar, int)
p = Ptr[T](n)
if atomic(T):
_read_raw(jar, p.as_byte(), n * sizeof(T))
else:
for i in range(n):
p[i] = unpickle(jar, T)
return DynamicTuple[T](p, n)
2021-09-27 14:02:44 -04:00
@extend
class Dict:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
import internal.khash as khash
2022-01-24 11:06:34 +01:00
2021-09-27 14:02:44 -04:00
if atomic(K) and atomic(V):
pickle(self._n_buckets, jar)
pickle(self._size, jar)
pickle(self._n_occupied, jar)
pickle(self._upper_bound, jar)
fsize = khash.__ac_fsize(self._n_buckets) if self._n_buckets > 0 else 0
_write_raw(jar, self._flags.as_byte(), fsize * sizeof(u32))
_write_raw(jar, self._keys.as_byte(), self._n_buckets * sizeof(K))
_write_raw(jar, self._vals.as_byte(), self._n_buckets * sizeof(V))
else:
pickle(self._n_buckets, jar)
size = len(self)
pickle(size, jar)
2022-01-24 11:06:34 +01:00
for k, v in self.items():
2021-09-27 14:02:44 -04:00
pickle(k, jar)
pickle(v, jar)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> Dict[K, V]:
2021-09-27 14:02:44 -04:00
import internal.khash as khash
2022-01-24 11:06:34 +01:00
2022-02-16 16:51:16 +01:00
d = {}
2021-09-27 14:02:44 -04:00
if atomic(K) and atomic(V):
n_buckets = unpickle(jar, int)
size = unpickle(jar, int)
n_occupied = unpickle(jar, int)
upper_bound = unpickle(jar, int)
fsize = khash.__ac_fsize(n_buckets) if n_buckets > 0 else 0
flags = Ptr[u32](fsize)
keys = Ptr[K](n_buckets)
vals = Ptr[V](n_buckets)
_read_raw(jar, flags.as_byte(), fsize * sizeof(u32))
_read_raw(jar, keys.as_byte(), n_buckets * sizeof(K))
_read_raw(jar, vals.as_byte(), n_buckets * sizeof(V))
d._n_buckets = n_buckets
d._size = size
d._n_occupied = n_occupied
d._upper_bound = upper_bound
d._flags = flags
d._keys = keys
d._vals = vals
else:
n_buckets = unpickle(jar, int)
size = unpickle(jar, int)
d.resize(n_buckets)
i = 0
while i < size:
k = unpickle(jar, K)
v = unpickle(jar, V)
d[k] = v
i += 1
return d
@extend
class Set:
2022-02-16 16:51:16 +01:00
def __pickle__(self, jar: Jar):
2021-09-27 14:02:44 -04:00
import internal.khash as khash
2022-01-24 11:06:34 +01:00
2021-09-27 14:02:44 -04:00
if atomic(K):
pickle(self._n_buckets, jar)
pickle(self._size, jar)
pickle(self._n_occupied, jar)
pickle(self._upper_bound, jar)
fsize = khash.__ac_fsize(self._n_buckets) if self._n_buckets > 0 else 0
_write_raw(jar, self._flags.as_byte(), fsize * sizeof(u32))
_write_raw(jar, self._keys.as_byte(), self._n_buckets * sizeof(K))
else:
pickle(self._n_buckets, jar)
size = len(self)
pickle(size, jar)
for k in self:
pickle(k, jar)
2022-01-24 11:06:34 +01:00
def __unpickle__(jar: Jar) -> Set[K]:
2021-09-27 14:02:44 -04:00
import internal.khash as khash
2022-01-24 11:06:34 +01:00
2022-02-16 16:51:16 +01:00
s = set[K]()
2021-09-27 14:02:44 -04:00
if atomic(K):
n_buckets = unpickle(jar, int)
size = unpickle(jar, int)
n_occupied = unpickle(jar, int)
upper_bound = unpickle(jar, int)
fsize = khash.__ac_fsize(n_buckets) if n_buckets > 0 else 0
flags = Ptr[u32](fsize)
keys = Ptr[K](n_buckets)
_read_raw(jar, flags.as_byte(), fsize * sizeof(u32))
_read_raw(jar, keys.as_byte(), n_buckets * sizeof(K))
s._n_buckets = n_buckets
s._size = size
s._n_occupied = n_occupied
s._upper_bound = upper_bound
s._flags = flags
s._keys = keys
else:
n_buckets = unpickle(jar, int)
size = unpickle(jar, int)
s.resize(n_buckets)
i = 0
while i < size:
k = unpickle(jar, K)
s.add(k)
i += 1
return s