Skip to content

Commit

Permalink
Minor fixes to support sticky
Browse files Browse the repository at this point in the history
  • Loading branch information
@udesou
udesou committed Feb 13, 2025
1 parent 7ea2484 commit 14f5c46
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 175 deletions.
84 changes: 7 additions & 77 deletions src/gc-mmtk.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,7 @@ inline void mmtk_set_side_metadata(const void* side_metadata_base, void* obj) {
STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) {
mmtk_set_side_metadata(MMTK_SIDE_LOG_BIT_BASE_ADDRESS, obj);
}
}

JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int osize, size_t align, void *ty)
Expand Down Expand Up @@ -1094,6 +1095,11 @@ void jl_gc_notify_image_load(const char* img_data, size_t len)
mmtk_set_vm_space((void*)img_data, len);
}

void jl_gc_notify_image_alloc(const char* img_data, size_t len)
{
mmtk_immortal_region_post_alloc((void*)img_data, len);
}

// ========================================================================= //
// Code specific to stock that is not supported by MMTk
// ========================================================================= //
Expand Down Expand Up @@ -1143,7 +1149,7 @@ JL_DLLEXPORT void jl_gc_queue_root(const struct _jl_value_t *ptr) JL_NOTSAFEPOIN
{
jl_task_t *ct = jl_current_task;
jl_ptls_t ptls = ct->ptls;
mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, parent, (const void*) 0);
mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, ptr, (const void*) 0);
}

JL_DLLEXPORT void jl_gc_queue_multiroot(const struct _jl_value_t *root, const void *stored,
Expand Down Expand Up @@ -1225,82 +1231,6 @@ JL_DLLEXPORT jl_value_t *jl_gc_internal_obj_base_ptr(void *p)
return NULL;
}

JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destdata,
jl_genericmemory_t *src, char* srcdata,
size_t n) JL_NOTSAFEPOINT
{
jl_datatype_t *dt = (jl_datatype_t*)jl_typetagof(dest);
if (dt != (jl_datatype_t*)jl_typetagof(src))
jl_exceptionf(jl_argumenterror_type, "jl_genericmemory_copyto requires source and dest to have same type");
const jl_datatype_layout_t *layout = dt->layout;
if (layout->flags.arrayelem_isboxed) {
_Atomic(void*) * dest_p = (_Atomic(void*)*)destdata;
_Atomic(void*) * src_p = (_Atomic(void*)*)srcdata;
jl_value_t *owner = jl_genericmemory_owner(dest);
jl_gc_wb(owner, NULL);
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
ssize_t done = 0;
if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) {
if (dest_p < src_p || dest_p > src_p + n) {
for (; done < n; done++) { // copy forwards
void *val = jl_atomic_load_relaxed(src_p + done);
jl_atomic_store_release(dest_p + done, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
jl_gc_queue_root(owner);
break;
}
}
src_p += done;
dest_p += done;
} else {
for (; done < n; done++) { // copy backwards
void *val = jl_atomic_load_relaxed(src_p + n - done - 1);
jl_atomic_store_release(dest_p + n - done - 1, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
jl_gc_queue_root(owner);
break;
}
}
}
n -= done;
}
}
return memmove_refs(dest_p, src_p, n);
}
size_t elsz = layout->size;
char *src_p = srcdata;
int isbitsunion = layout->flags.arrayelem_isunion;
if (isbitsunion) {
char *sourcetypetagdata = jl_genericmemory_typetagdata(src);
char *desttypetagdata = jl_genericmemory_typetagdata(dest);
memmove(desttypetagdata+(size_t)destdata, sourcetypetagdata+(size_t)srcdata, n);
srcdata = (char*)src->ptr + elsz*(size_t)srcdata;
destdata = (char*)dest->ptr + elsz*(size_t)destdata;
}
if (layout->first_ptr != -1) {
memmove_refs((_Atomic(void*)*)destdata, (_Atomic(void*)*)srcdata, n * elsz / sizeof(void*));
jl_value_t *owner = jl_genericmemory_owner(dest);
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) {
dt = (jl_datatype_t*)jl_tparam1(dt);
for (size_t done = 0; done < n; done++) { // copy forwards
char* s = (char*)src_p+done*elsz;
if (*((jl_value_t**)s+layout->first_ptr) != NULL)
jl_gc_queue_multiroot(owner, s, dt);
}
}
}
}
else {
memmove(destdata, srcdata, n * elsz);
}
}


#ifdef __cplusplus
}
#endif
14 changes: 14 additions & 0 deletions src/gc-wb-mmtk.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ extern "C" {

extern void mmtk_object_reference_write_post(void* mutator, const void* parent, const void* ptr);
extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, const void* ptr);
extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS;

#define MMTK_OBJECT_BARRIER (1)
// Stickyimmix needs write barrier. Immix does not need write barrier.
Expand Down Expand Up @@ -64,6 +65,19 @@ STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_
mmtk_gc_wb_fast(parent, (void*)0);
}

STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
jl_genericmemory_t *src, _Atomic(void*) * src_p,
size_t* n) JL_NOTSAFEPOINT
{
mmtk_gc_wb_fast(dest_owner, (void*)0);
}

STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
{
mmtk_gc_wb_fast(owner, (void*)0);
}


#ifdef __cplusplus
}
Expand Down
54 changes: 54 additions & 0 deletions src/gc-wb-stock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,60 @@ STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
}

STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
jl_genericmemory_t *src, _Atomic(void*) * src_p,
size_t* n) JL_NOTSAFEPOINT
{
if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
size_t done = 0;
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
if (dest_p < src_p || dest_p > src_p + (*n)) {
for (; done < (*n); done++) { // copy forwards
void *val = jl_atomic_load_relaxed(src_p + done);
jl_atomic_store_release(dest_p + done, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
jl_gc_queue_root(dest_owner);
break;
}
}
src_p += done;
dest_p += done;
}
else {
for (; done < (*n); done++) { // copy backwards
void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1);
jl_atomic_store_release(dest_p + (*n) - done - 1, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
jl_gc_queue_root(dest_owner);
break;
}
}
}
(*n) -= done;
}
}
}

STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
{
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
size_t elsz = dt->layout->size;
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
dt = (jl_datatype_t*)jl_tparam1(dt);
for (size_t done = 0; done < n; done++) { // copy forwards
char* s = (char*)src_p+done*elsz;
if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL)
jl_gc_queue_multiroot(owner, s, dt);
}
}
}
}

#ifdef __cplusplus
}
#endif
Expand Down
108 changes: 12 additions & 96 deletions src/julia.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1177,12 +1177,6 @@ JL_DLLEXPORT void jl_free_stack(void *stkbuf, size_t bufsz);
// thread-local allocator of the current thread.
JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value);

#ifndef MMTK_GC
#include "gc-wb-stock.h"
#else
#include "gc-wb-mmtk.h"
#endif

JL_DLLEXPORT void jl_gc_safepoint(void);
JL_DLLEXPORT int jl_safepoint_suspend_thread(int tid, int waitstate);
JL_DLLEXPORT void jl_safepoint_suspend_all_threads(struct _jl_task_t *ct);
Expand Down Expand Up @@ -1245,6 +1239,18 @@ STATIC_INLINE jl_value_t *jl_svecset(
#define jl_array_maxsize(a) (((jl_array_t*)(a))->ref.mem->length)
#define jl_array_len(a) (jl_array_ndims(a) == 1 ? jl_array_nrows(a) : jl_array_maxsize(a))

JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT;
#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack))

STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;

// write barriers
#ifndef MMTK_GC
#include "gc-wb-stock.h"
#else
#include "gc-wb-mmtk.h"
#endif

/*
how - allocation style
0 = data is inlined
Expand Down Expand Up @@ -1301,94 +1307,6 @@ STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set(
}
#endif

// GC write barriers

STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT
{
// parent and ptr isa jl_value_t*
if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset
(jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young
jl_gc_queue_root((jl_value_t*)parent);
}

STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t*
{
// if ptr is old
if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) {
jl_gc_queue_root((jl_value_t*)ptr);
}
}

STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT
{
// 3 == GC_OLD_MARKED
// ptr is an immutable object
if (__likely(jl_astaggedvalue(parent)->bits.gc != 3))
return; // parent is young or in remset
if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3))
return; // ptr is old and not in remset (thus it does not point to young)
jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr);
const jl_datatype_layout_t *ly = dt->layout;
if (ly->npointers)
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
}

STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;

STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
jl_genericmemory_t *src, _Atomic(void*) * src_p,
size_t* n) JL_NOTSAFEPOINT
{
if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
size_t done = 0;
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
if (dest_p < src_p || dest_p > src_p + (*n)) {
for (; done < (*n); done++) { // copy forwards
void *val = jl_atomic_load_relaxed(src_p + done);
jl_atomic_store_release(dest_p + done, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
jl_gc_queue_root(dest_owner);
break;
}
}
src_p += done;
dest_p += done;
}
else {
for (; done < (*n); done++) { // copy backwards
void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1);
jl_atomic_store_release(dest_p + (*n) - done - 1, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
jl_gc_queue_root(dest_owner);
break;
}
}
}
(*n) -= done;
}
}
}

STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
{
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
size_t elsz = dt->layout->size;
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
dt = (jl_datatype_t*)jl_tparam1(dt);
for (size_t done = 0; done < n; done++) { // copy forwards
char* s = (char*)src_p+done*elsz;
if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL)
jl_gc_queue_multiroot(owner, s, dt);
}
}
}
}

STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT
{
jl_genericmemory_t *m_ = (jl_genericmemory_t*)m;
Expand Down Expand Up @@ -2401,8 +2319,6 @@ JL_DLLEXPORT void JL_NORETURN jl_throw(jl_value_t *e JL_MAYBE_UNROOTED);
JL_DLLEXPORT void JL_NORETURN jl_rethrow(void);
JL_DLLEXPORT void JL_NORETURN jl_rethrow_other(jl_value_t *e JL_MAYBE_UNROOTED);
JL_DLLEXPORT void JL_NORETURN jl_no_exc_handler(jl_value_t *e, jl_task_t *ct);
JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT;
#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack))

extern JL_DLLIMPORT int jl_task_gcstack_offset;
extern JL_DLLIMPORT int jl_task_ptls_offset;
Expand Down
6 changes: 4 additions & 2 deletions src/staticdata.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4166,9 +4166,10 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i
char *sysimg;
int success = !needs_permalloc;
ios_seek(f, datastartpos);
if (needs_permalloc)
if (needs_permalloc) {
sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0);
else
jl_gc_notify_image_alloc(sysimg, len);
} else
sysimg = &f->buf[f->bpos];
if (needs_permalloc)
success = ios_readall(f, sysimg, len) == len;
Expand Down Expand Up @@ -4291,6 +4292,7 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname)
ios_seek_end(&f);
size_t len = ios_pos(&f);
char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0);
jl_gc_notify_image_alloc(sysimg, len);
ios_seek(&f, 0);
if (ios_readall(&f, sysimg, len) != len)
jl_errorf("Error reading system image file.");
Expand Down

0 comments on commit 14f5c46

Please sign in to comment.