I want to parse JSON file (>1GB) from C code (because of speed), and it looks like many invocations of caml_alloc_tuple lead me to “Fatal error: allocation failure during minor GC”.
There is a function caml_alloc_shr in the manual but it looks like it is not the one: it allocated large blocks in another place, not everything. (Also, it crashes my program, but I hope it is not relevant.)
This is not supposed to happen. How are you calling it?
Backtrace
Fatal error: allocation failure during minor GC
Program received signal SIGABRT, Aborted.
0x00007ffff78a5fed in pthread_kill () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) bt
#0 0x00007ffff78a5fed in pthread_kill () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007ffff7845e2e in raise () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007ffff7828888 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x0000555555936ce5 in caml_fatal_error (
msg=msg@entry=0x555555958ab8 “allocation failure during minor GC”) at runtime/misc.c:116
#4 0x0000555555935831 in alloc_shared (reserved=, d=,
wosize=, tag=) at runtime/minor_gc.c:159
#5 oldify_one (st_v=st_v@entry=0x7fffffffc810, v=, p=0x7ffff7fa1b50)
at runtime/minor_gc.c:278
#6 0x0000555555936346 in caml_empty_minor_heap_promote (domain=domain@entry=0x555555ad86d0,
participating_count=participating_count@entry=1,
participating=participating@entry=0x555555ad7eb0) at runtime/minor_gc.c:574
#7 0x0000555555936469 in caml_stw_empty_minor_heap_no_major_slice (domain=0x555555ad86d0,
participating_count=1, participating=0x555555ad7eb0, unused=)
at runtime/minor_gc.c:798
#8 0x000055555591d9ad in caml_try_run_on_all_domains_with_spin_work (sync=sync@entry=1,
handler=handler@entry=0x555555936630 <caml_stw_empty_minor_heap>, data=data@entry=0x0,
leader_setup=leader_setup@entry=0x555555935230 <caml_empty_minor_heap_setup>,
enter_spin_callback=enter_spin_callback@entry=0x555555935260 <caml_do_opportunistic_major_slice>, enter_spin_data=enter_spin_data@entry=0x0) at runtime/domain.c:1695
#9 0x0000555555936772 in caml_try_empty_minor_heap_on_all_domains () at runtime/minor_gc.c:858
#10 caml_empty_minor_heaps_once () at runtime/minor_gc.c:879
#11 0x000055555591d560 in caml_poll_gc_work () at runtime/domain.c:1868
#12 0x0000555555936805 in caml_alloc_small_dispatch (dom_st=dom_st@entry=0x555555ad86d0,
wosize=wosize@entry=2, flags=flags@entry=1, nallocs=nallocs@entry=1,
encoded_alloc_lens=encoded_alloc_lens@entry=0x0) at runtime/minor_gc.c:906
#13 0x0000555555913318 in caml_alloc_string (len=len@entry=11) at runtime/alloc.c:183
#14 0x000055555591335e in caml_alloc_initialized_string (len=11, p=0x7fffffffcc30 “type_struct”)
at runtime/alloc.c:197
#15 caml_copy_string (s=0x7fffffffcc30 “type_struct”) at runtime/alloc.c:218
#16 0x00005555558d7e68 in convert (j=…) at simdjson_stubs.cxx:93
#17 0x00005555558d79fb in convert (j=…) at simdjson_stubs.cxx:63
#18 0x00005555558d83fa in caml_read_json (_string=93824996856552) at simdjson_stubs.cxx:128
My code
#include <stdio.h>
#include <iterator>
#include <simdjson.h>
extern "C" {
#include <caml/mlvalues.h>
#include <caml/alloc.h>
#include <caml/memory.h>
#define MAKE_VARIANT(_cell, size, name, body) \
{ _cell = caml_alloc_tuple(size); \
Store_field(_cell, 0, caml_hash_variant(name)); \
Store_field(_cell, 1, body); }
value convert(simdjson::dom::element j) {
using namespace simdjson::dom;
CAMLparam0();
CAMLlocal3(_ans, _cell, _cell2 );
switch (j.type()) {
case element_type::NULL_VALUE:
CAMLreturn(caml_hash_variant("Null"));
break;
case element_type::INT64:{
simdjson::simdjson_result< int64_t > r = j.get_int64();
// std::cout << r.error() << std::endl;
auto n = r.value();
// std::cout << "Int" << n<< std::endl;
MAKE_VARIANT(_ans, 2, "Int", Val_int(n));
CAMLreturn(_ans);
break;
}
case element_type::BOOL: {
simdjson::simdjson_result< bool > r = j.get_bool();
auto b = r.value();
MAKE_VARIANT(_ans, 2, "Bool", Val_int(b));
CAMLreturn(_ans);
break;
}
case element_type::DOUBLE: {
double r = j.get_double().value();
MAKE_VARIANT(_ans, 2, "Float", caml_copy_double(r));
CAMLreturn(_ans);
break;
}
case element_type::STRING:{
auto s = j.get_c_str().value();
// std::cout << " String gotten: " << s << std::endl;
MAKE_VARIANT(_ans, 2, "String", caml_copy_string(s) );
CAMLreturn(_ans);
break;
}
case simdjson::dom::element_type::ARRAY: {
auto vec = j.get_array().value();
std::vector<value> answers;
for (auto rit = vec.begin(); rit != vec.end(); ++rit) {
const simdjson::dom::element p = *rit;
answers.push_back( convert(p) );
}
// std::cout << "size = " << answers.size() << std::endl;
_ans = Val_int(0); // []
for (auto i = answers.rbegin(); i != answers.rend(); ++i) {
_cell = caml_alloc_tuple(2); // ::
Store_field(_cell, 0, *i);
Store_field(_cell, 1, _ans);
_ans = _cell;
}
_cell = caml_alloc_tuple(2);
MAKE_VARIANT(_cell, 2, "List", _ans);
CAMLreturn(_cell);
break;
}
case simdjson::dom::element_type::OBJECT: {
// std::cout << "object\n";
auto vec = j.get_object().value();
std::vector<std::pair<std::string_view, value>> answers;
for (auto rit = vec.begin(); rit != vec.end(); ++rit) {
const simdjson::dom::key_value_pair p = *rit;
answers.push_back(std::make_pair(p.key, convert(p.value)));
// std::cout << p.key << " ";
}
// std::cout << "size = " << answers.size() << std::endl;
_ans = Val_int(0); // []
for (auto i = answers.rbegin(); i != answers.rend(); ++i) {
std::pair<std::string_view, value> p = *i;
_cell2 = caml_alloc_tuple(2);
Store_field(_cell2, 0, caml_copy_string(std::string(p.first).c_str()) );
Store_field(_cell2, 1, p.second);
_cell = caml_alloc_tuple(2); // ::
Store_field(_cell, 0, _cell2);
Store_field(_cell, 1, _ans);
_ans = _cell;
}
_cell = caml_alloc_tuple(2);
MAKE_VARIANT(_cell, 2, "Assoc", _ans);
CAMLreturn(_cell);
break;
}
case simdjson::dom::element_type::BIGINT:
std::cout << "BigINT not supported" << std::endl;
exit(1);
break;
}
std::cout << "Unsupported type of simdjson node" << std::endl;
exit(1);
}
CAMLprim value
caml_read_json(value _string)
{
CAMLparam1(_string);
CAMLlocal1(_ans);
simdjson::dom::parser parser;
simdjson::dom::element tweets;
auto error = parser.load(String_val(_string)).get(tweets);
if (error) { std::cerr << error << std::endl; exit(EXIT_FAILURE); }
_ans = convert(tweets);
CAMLreturn(_ans);
}
}
Hum. Any chance you are running out of RAM during the execution?
Aside of that, to answer your original question, yes, you can allocate directly in the major heap using caml_alloc_shr (in which case you must initialize the fields with caml_initialize). Performance-wise it’s rarely a gain, but it may help you diagnose what is going wrong.
I have some doubts that allocating a pure OCaml structure would be faster in C(++) than in OCaml, but in any case your code is clearly wrong because you have several occurrences of temporary expressions of type value not registered with the GC. The calls to caml_copy_double and caml_copy_string are borderline (their result should be registered, but you might end up in a case where they’re not live across another allocation so you won’t see bugs). Storing the result of convert inside the answers vector is clearly going to cause problems and is likely the cause for your error.
1 Like
You are referencing tons of OCaml valuers in your anwers vectors, which the GC is not aware of.
std::vector<value> answers;
std::vector<std::pair<std::string_view, value>> answers;
You should set up caml_scan_roots_hook to manually traverse these vectors during garbage collection.