MinaProtocol / mina / 366

Committed 13 Jul 2025 09:41AM UTC coverage: 32.781% (-28.2%) from 60.949%

Build # 366

Build Type

push

buildkite

Committed by

dkijania

Commit Message

add lmdb dependency to debian

Run Details

23599 of 71989 relevant lines covered (32.78%)

24555.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.33

/src/lib/disk_cache/lmdb/disk_cache.ml

(* Cache proofs using the lmdb *)

open Core
open Lmdb_storage.Generic

module Make (Data : Binable.S) = struct
  module F (Db : Db) = struct
    type holder = (int, Data.t) Db.t

    let mk_maps { Db.create } =
      create Lmdb_storage.Conv.uint32_be
        (Lmdb_storage.Conv.bin_prot_conv Data.bin_t)

    let config = { default_config with initial_mmap_size = 256 lsl 20 }
  end

  module Rw = Read_write (F)

  type t =
    { env : Rw.t
    ; db : Rw.holder
    ; counter : int ref
    ; garbage : int Hash_set.t
          (** A list of ids that are no longer reachable from OCaml's side *)
    }

  (** How big can the above hashset be before we do a cleanup *)
  let garbage_size_limit = 512

  let initialize path ~logger =
    Async.Deferred.Result.map (Disk_cache_utils.initialize_dir path ~logger)
      ~f:(fun path ->
        let env, db = Rw.create path in
        { env; db; counter = ref 0; garbage = Hash_set.create (module Int) } )

  type id = { idx : int }

  let get ({ env; db; _ } : t) ({ idx } : id) : Data.t =
    Rw.get ~env db idx |> Option.value_exn

  let put ({ env; db; counter; garbage } : t) (x : Data.t) : id =
    (* TODO: we may reuse IDs by pulling them from the `garbage` hash set *)
    let idx = !counter in
    incr counter ;
    let res = { idx } in
    (* When this reference is GC'd, delete the file. *)
    Gc.Expert.add_finalizer_last_exn res (fun () ->
        (* The actual deletion is delayed, as GC maybe triggered in LMDB's
           critical section. LMDB critical section then will be re-entered if
           it's invoked directly in a GC hook.
           This causes mutex double-acquiring and node freezes. *)
        Hash_set.add garbage idx ) ;
    if Hash_set.length garbage >= garbage_size_limit then (
      Hash_set.iter garbage ~f:(fun to_remove -> Rw.remove ~env db to_remove) ;
      Hash_set.clear garbage ) ;
    Rw.set ~env db idx x ;
    res

  let iteri ({ env; db; _ } : t) ~f = Rw.iter ~env db ~f

  let count ({ env; db; _ } : t) =
    let sum = ref 0 in
    Rw.iter ~env db ~f:(fun _ _ -> incr sum ; `Continue) ;
    !sum

  let int_of_id { idx } = idx
end

let%test_module "disk_cache lmdb" =
  ( module struct
    include Disk_cache_test_lib.Make_extended (Make)

    let%test_unit "remove data on gc" = remove_data_on_gc ~gc_strict:false ()

    let%test_unit "simple read/write (with iteration)" =
      simple_write_with_iteration ()

    let%test_unit "initialization special cases" =
      initialization_special_cases ()
  end )

1	(* Cache proofs using the lmdb *)
2		6✔
3	open Core
4	open Lmdb_storage.Generic
5
6	module Make (Data : Binable.S) = struct
7	module F (Db : Db) = struct
8	type holder = (int, Data.t) Db.t
9
10	let mk_maps { Db.create } =
11	create Lmdb_storage.Conv.uint32_be	×
12	(Lmdb_storage.Conv.bin_prot_conv Data.bin_t)	×
13
14	let config = { default_config with initial_mmap_size = 256 lsl 20 }
15	end
16
17	module Rw = Read_write (F)
18
19	type t =
20	{ env : Rw.t
21	; db : Rw.holder
22	; counter : int ref
23	; garbage : int Hash_set.t
24	(** A list of ids that are no longer reachable from OCaml's side *)
25	}
26
27	(** How big can the above hashset be before we do a cleanup *)
28	let garbage_size_limit = 512
29
30	let initialize path ~logger =
31	Async.Deferred.Result.map (Disk_cache_utils.initialize_dir path ~logger)	×
32	~f:(fun path ->
33	let env, db = Rw.create path in	×
34	{ env; db; counter = ref 0; garbage = Hash_set.create (module Int) } )	×
35
36	type id = { idx : int }
37
38	let get ({ env; db; _ } : t) ({ idx } : id) : Data.t =
39	Rw.get ~env db idx \|> Option.value_exn	×
40
41	let put ({ env; db; counter; garbage } : t) (x : Data.t) : id =
42	(* TODO: we may reuse IDs by pulling them from the `garbage` hash set *)
43	let idx = !counter in	×
44	incr counter ;
45	let res = { idx } in	×
46	(* When this reference is GC'd, delete the file. *)
47	Gc.Expert.add_finalizer_last_exn res (fun () ->
48	(* The actual deletion is delayed, as GC maybe triggered in LMDB's
49	critical section. LMDB critical section then will be re-entered if
50	it's invoked directly in a GC hook.
51	This causes mutex double-acquiring and node freezes. *)
52	Hash_set.add garbage idx ) ;	×
53	if Hash_set.length garbage >= garbage_size_limit then (	×
54	Hash_set.iter garbage ~f:(fun to_remove -> Rw.remove ~env db to_remove) ;	×
55	Hash_set.clear garbage ) ;	×
56	Rw.set ~env db idx x ;	×
57	res	×
58
59	let iteri ({ env; db; _ } : t) ~f = Rw.iter ~env db ~f	×
60
61	let count ({ env; db; _ } : t) =
62	let sum = ref 0 in	×
63	Rw.iter ~env db ~f:(fun _ _ -> incr sum ; `Continue) ;	×
64	!sum	×
65
66	let int_of_id { idx } = idx	×
67	end
68
69	let%test_module "disk_cache lmdb" =
70	( module struct
71	include Disk_cache_test_lib.Make_extended (Make)
72
73	let%test_unit "remove data on gc" = remove_data_on_gc ~gc_strict:false ()	×
74
75	let%test_unit "simple read/write (with iteration)" =
76	simple_write_with_iteration ()	×
77
78	let%test_unit "initialization special cases" =
79	initialization_special_cases ()	×
80	end )	12✔

MinaProtocol / mina / 366

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous