MinaProtocol / mina / 431

Committed 29 Jul 2025 03:33PM UTC coverage: 32.152% (-1.1%) from 33.242%

Build # 431

Build Type

push

buildkite

Committed by

web-flow

Commit Message

Merge pull request #17590 from MinaProtocol/georgeee/merge-compatible-to-develop-29jul2025

Merge compatible to develop (29 Jul 2025)

Run Details

80 of 143 new or added lines in 20 files covered. (55.94%)

655 existing lines in 25 files now uncovered.

23110 of 71877 relevant lines covered (32.15%)

24858.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.6

/src/lib/transition_frontier/persistent_root/persistent_root.ml

open Core
open Mina_base
module Ledger = Mina_ledger.Ledger
open Frontier_base
module Ledger_transfer = Mina_ledger.Ledger_transfer.Make (Ledger) (Ledger.Db)
module Ledger_transfer_stable =
  Mina_ledger.Ledger_transfer.Make (Ledger.Db) (Ledger.Db)

let transfer_snarked_root =
  Ledger.Root.transfer_accounts_with
    ~stable:Ledger_transfer_stable.transfer_accounts

let genesis_root_identifier ~genesis_state_hash =
  let open Root_identifier.Stable.Latest in
  { state_hash = genesis_state_hash }

let with_file ?size filename access_level ~f =
  let open Unix in
  let shared, mode =
    match access_level with
    | `Read ->
        (false, [ O_RDONLY ])
    | `Write ->
        (true, [ O_RDWR; O_TRUNC; O_CREAT ])
  in
  let fd = Unix.openfile filename ~mode in
  let buf_size =
    match size with
    | None ->
        Int64.to_int_exn Unix.(fstat fd).st_size
    | Some sz ->
        sz
  in
  (* Bigstring.map_file has been removed. We copy its old implementation. *)
  let buf =
    Bigarray.(
      array1_of_genarray
        (Core.Unix.map_file fd char c_layout ~shared [| buf_size |]))
  in
  let x = f buf in
  Bigstring.unsafe_destroy buf ;
  Unix.close fd ;
  x

module Locations = struct
  let snarked_ledger root = Filename.concat root "snarked_ledger"

  let tmp_snarked_ledger root = Filename.concat root "tmp_snarked_ledger"

  (** potential_snarked_ledgers is a json file that stores a list of potential
      snarked ledgeres *)
  let potential_snarked_ledgers root =
    Filename.concat root "potential_snarked_ledgers.json"

  (** potential_snarked_ledger is the actual location of each potential snarked
      ledger *)
  let potential_snarked_ledger root =
    let uuid = Uuid_unix.create () in
    Filename.concat root ("snarked_ledger" ^ Uuid.to_string_hum uuid)

  let root_identifier root = Filename.concat root "root"
end

(* TODO: create a reusable singleton factory abstraction *)
module rec Instance_type : sig
  type t =
    { snarked_ledger : Ledger.Root.t
    ; potential_snarked_ledgers : string Queue.t
    ; factory : Factory_type.t
    }
end =
  Instance_type

and Factory_type : sig
  type t =
    { directory : string
    ; logger : Logger.t
    ; mutable instance : Instance_type.t option
    ; ledger_depth : int
    }
end =
  Factory_type

open Instance_type
open Factory_type

module Instance = struct
  type t = Instance_type.t

  let potential_snarked_ledgers_to_yojson queue =
    `List (List.map (Queue.to_list queue) ~f:(fun filename -> `String filename))

  let potential_snarked_ledgers_of_yojson json =
    Yojson.Safe.Util.to_list json |> List.map ~f:Yojson.Safe.Util.to_string

  let load_potential_snarked_ledgers_from_disk factory =
    let location = Locations.potential_snarked_ledgers factory.directory in
    if phys_equal (Sys.file_exists location) `Yes then
      Yojson.Safe.from_file location |> potential_snarked_ledgers_of_yojson
    else []

  let write_potential_snarked_ledgers_to_disk t =
    Yojson.Safe.to_file
      (Locations.potential_snarked_ledgers t.factory.directory)
      (potential_snarked_ledgers_to_yojson t.potential_snarked_ledgers)

  let enqueue_snarked_ledger ~location t =
    Queue.enqueue t.potential_snarked_ledgers location ;
    write_potential_snarked_ledgers_to_disk t

  let dequeue_snarked_ledger t =
    let location = Queue.dequeue_exn t.potential_snarked_ledgers in
    Mina_stdlib_unix.File_system.rmrf location ;
    write_potential_snarked_ledgers_to_disk t

  let destroy t =
    List.iter
      (Queue.to_list t.potential_snarked_ledgers)
      ~f:Mina_stdlib_unix.File_system.rmrf ;
    Mina_stdlib_unix.File_system.rmrf
      (Locations.potential_snarked_ledgers t.factory.directory) ;
    Ledger.Root.close t.snarked_ledger ;
    t.factory.instance <- None

  let close t =
    Ledger.Root.close t.snarked_ledger ;
    t.factory.instance <- None

  let create factory =
    let snarked_ledger =
      Ledger.Root.create_single ~depth:factory.ledger_depth
        ~directory_name:(Locations.snarked_ledger factory.directory)
        ()
    in
    { snarked_ledger; potential_snarked_ledgers = Queue.create (); factory }

  (** When we load from disk,
      1. Check the potential_snarked_ledgers to see if any one of these
         matches the snarked_ledger_hash in persistent_frontier;
      2. if none of those works, we load the old snarked_ledger and check if
         the old snarked_ledger matches with persistent_frontier;
      3. if not, we just reset all the persisted data and start from genesis
   *)
  let load_from_disk factory ~snarked_ledger_hash ~logger =
    let potential_snarked_ledgers =
      load_potential_snarked_ledgers_from_disk factory
    in
    let snarked_ledger =
      List.fold_until potential_snarked_ledgers ~init:None
        ~f:(fun _ location ->
          let potential_snarked_ledger =
            Ledger.Root.create_single ~depth:factory.ledger_depth
              ~directory_name:location ()
          in
          let potential_snarked_ledger_hash =
            Frozen_ledger_hash.of_ledger_hash
            @@ Ledger.Root.merkle_root potential_snarked_ledger
          in
          [%log debug]
            ~metadata:
              [ ( "potential_snarked_ledger_hash"
                , Frozen_ledger_hash.to_yojson potential_snarked_ledger_hash )
              ]
            "loaded potential_snarked_ledger from disk" ;
          if
            Frozen_ledger_hash.equal potential_snarked_ledger_hash
              snarked_ledger_hash
          then (
            let snarked_ledger =
              Ledger.Root.create_single ~depth:factory.ledger_depth
                ~directory_name:(Locations.tmp_snarked_ledger factory.directory)
                ()
            in
            match
              transfer_snarked_root ~src:potential_snarked_ledger
                ~dest:snarked_ledger
            with
            | Ok _ ->
                Ledger.Root.close potential_snarked_ledger ;
                Mina_stdlib_unix.File_system.rmrf
                @@ Locations.snarked_ledger factory.directory ;
                Sys.rename
                  (Locations.tmp_snarked_ledger factory.directory)
                  (Locations.snarked_ledger factory.directory) ;
                List.iter potential_snarked_ledgers
                  ~f:Mina_stdlib_unix.File_system.rmrf ;
                Mina_stdlib_unix.File_system.rmrf
                  (Locations.potential_snarked_ledgers factory.directory) ;
                Stop (Some snarked_ledger)
            | Error e ->
                Ledger.Root.close potential_snarked_ledger ;
                List.iter potential_snarked_ledgers
                  ~f:Mina_stdlib_unix.File_system.rmrf ;
                Mina_stdlib_unix.File_system.rmrf
                  (Locations.potential_snarked_ledgers factory.directory) ;
                [%log' error factory.logger]
                  ~metadata:[ ("error", `String (Error.to_string_hum e)) ]
                  "Ledger_transfer failed" ;
                Stop None )
          else (
            Ledger.Root.close potential_snarked_ledger ;
            Continue None ) )
        ~finish:(fun _ ->
          List.iter potential_snarked_ledgers
            ~f:Mina_stdlib_unix.File_system.rmrf ;
          Mina_stdlib_unix.File_system.rmrf
            (Locations.potential_snarked_ledgers factory.directory) ;
          None )
    in
    match snarked_ledger with
    | None ->
        let snarked_ledger =
          Ledger.Root.create_single ~depth:factory.ledger_depth
            ~directory_name:(Locations.snarked_ledger factory.directory)
            ()
        in
        let potential_snarked_ledger_hash =
          Frozen_ledger_hash.of_ledger_hash
          @@ Ledger.Root.merkle_root snarked_ledger
        in
        if
          Frozen_ledger_hash.equal potential_snarked_ledger_hash
            snarked_ledger_hash
        then
          Ok
            { snarked_ledger
            ; potential_snarked_ledgers = Queue.create ()
            ; factory
            }
        else (
          Ledger.Root.close snarked_ledger ;
          Error `Snarked_ledger_mismatch )
    | Some snarked_ledger ->
        Ok
          { snarked_ledger
          ; potential_snarked_ledgers = Queue.create ()
          ; factory
          }

  let snarked_ledger { snarked_ledger; _ } = snarked_ledger

  let set_root_identifier t new_root_identifier =
    [%log' trace t.factory.logger]
      ~metadata:
        [ ("root_identifier", Root_identifier.to_yojson new_root_identifier) ]
      "Setting persistent root identifier" ;
    let size = Root_identifier.Stable.Latest.bin_size_t new_root_identifier in
    with_file (Locations.root_identifier t.factory.directory) `Write ~size
      ~f:(fun buf ->
        ignore
          ( Root_identifier.Stable.Latest.bin_write_t buf ~pos:0
              new_root_identifier
            : int ) )

  (* defaults to genesis *)
  let load_root_identifier t =
    let file = Locations.root_identifier t.factory.directory in
    match Unix.access file [ `Exists; `Read ] with
    | Error _ ->
        None
    | Ok () ->
        with_file file `Read ~f:(fun buf ->
            let root_identifier =
              Root_identifier.Stable.Latest.bin_read_t buf ~pos_ref:(ref 0)
            in
            [%log' trace t.factory.logger]
              ~metadata:
                [ ("root_identifier", Root_identifier.to_yojson root_identifier)
                ]
              "Loaded persistent root identifier" ;
            Some root_identifier )

  let set_root_state_hash t state_hash = set_root_identifier t { state_hash }
end

type t = Factory_type.t

let create ~logger ~directory ~ledger_depth =
  { directory; logger; instance = None; ledger_depth }

let create_instance_exn t =
  assert (Option.is_none t.instance) ;
  let instance = Instance.create t in
  t.instance <- Some instance ;
  instance

let load_from_disk_exn t ~snarked_ledger_hash ~logger =
  let open Result.Let_syntax in
  assert (Option.is_none t.instance) ;
  let%map instance = Instance.load_from_disk t ~snarked_ledger_hash ~logger in
  t.instance <- Some instance ;
  instance

let with_instance_exn t ~f =
  let instance = create_instance_exn t in
  let x = f instance in
  Instance.close instance ; x

let reset_to_genesis_exn t ~precomputed_values =
  assert (Option.is_none t.instance) ;
  Mina_stdlib_unix.File_system.rmrf t.directory ;
  with_instance_exn t ~f:(fun instance ->
      ignore
        ( Precomputed_values.populate_root precomputed_values
            (Instance.snarked_ledger instance)
          |> Or_error.map ~f:Ledger.Root.as_unmasked
          : Ledger.Any_ledger.witness Or_error.t ) ;
      Instance.set_root_identifier instance
        (genesis_root_identifier
           ~genesis_state_hash:
             (Precomputed_values.genesis_state_hashes precomputed_values)
               .state_hash ) )

1	open Core	49✔
2	open Mina_base
3	module Ledger = Mina_ledger.Ledger
4	open Frontier_base
5	module Ledger_transfer = Mina_ledger.Ledger_transfer.Make (Ledger) (Ledger.Db)
6	module Ledger_transfer_stable =
7	Mina_ledger.Ledger_transfer.Make (Ledger.Db) (Ledger.Db)
8
9	let transfer_snarked_root =
10	Ledger.Root.transfer_accounts_with
11	~stable:Ledger_transfer_stable.transfer_accounts
12
13	let genesis_root_identifier ~genesis_state_hash =
14	let open Root_identifier.Stable.Latest in	×
15	{ state_hash = genesis_state_hash }
16
17	let with_file ?size filename access_level ~f =
18	let open Unix in	×
19	let shared, mode =
20	match access_level with
21	\| `Read ->	×
22	(false, [ O_RDONLY ])
23	\| `Write ->	×
24	(true, [ O_RDWR; O_TRUNC; O_CREAT ])
25	in
26	let fd = Unix.openfile filename ~mode in
27	let buf_size =	×
28	match size with
29	\| None ->	×
30	Int64.to_int_exn Unix.(fstat fd).st_size	×
31	\| Some sz ->	×
32	sz
33	in
34	(* Bigstring.map_file has been removed. We copy its old implementation. *)
35	let buf =
36	Bigarray.(
37	array1_of_genarray	×
38	(Core.Unix.map_file fd char c_layout ~shared [\| buf_size \|]))	×
39	in
40	let x = f buf in
41	Bigstring.unsafe_destroy buf ;	×
42	Unix.close fd ;	×
43	x	×
44
45	module Locations = struct
46	let snarked_ledger root = Filename.concat root "snarked_ledger"	×
47
48	let tmp_snarked_ledger root = Filename.concat root "tmp_snarked_ledger"	×
49
50	(** potential_snarked_ledgers is a json file that stores a list of potential
51	snarked ledgeres *)
52	let potential_snarked_ledgers root =
53	Filename.concat root "potential_snarked_ledgers.json"	×
54
55	(** potential_snarked_ledger is the actual location of each potential snarked
56	ledger *)
57	let potential_snarked_ledger root =
58	let uuid = Uuid_unix.create () in	×
59	Filename.concat root ("snarked_ledger" ^ Uuid.to_string_hum uuid)	×
60
61	let root_identifier root = Filename.concat root "root"	×
62	end
63
64	(* TODO: create a reusable singleton factory abstraction *)
65	module rec Instance_type : sig
66	type t =
67	{ snarked_ledger : Ledger.Root.t
68	; potential_snarked_ledgers : string Queue.t
69	; factory : Factory_type.t
70	}
71	end =
72	Instance_type
73
74	and Factory_type : sig
75	type t =
76	{ directory : string
77	; logger : Logger.t
78	; mutable instance : Instance_type.t option
79	; ledger_depth : int
80	}
81	end =
82	Factory_type
83
84	open Instance_type
85	open Factory_type
86
87	module Instance = struct
88	type t = Instance_type.t
89
90	let potential_snarked_ledgers_to_yojson queue =
91	`List (List.map (Queue.to_list queue) ~f:(fun filename -> `String filename))	×
92
93	let potential_snarked_ledgers_of_yojson json =
94	Yojson.Safe.Util.to_list json \|> List.map ~f:Yojson.Safe.Util.to_string	×
95
96	let load_potential_snarked_ledgers_from_disk factory =
97	let location = Locations.potential_snarked_ledgers factory.directory in	×
98	if phys_equal (Sys.file_exists location) `Yes then	×
99	Yojson.Safe.from_file location \|> potential_snarked_ledgers_of_yojson	×
100	else []	×
101
102	let write_potential_snarked_ledgers_to_disk t =
103	Yojson.Safe.to_file	×
104	(Locations.potential_snarked_ledgers t.factory.directory)	×
105	(potential_snarked_ledgers_to_yojson t.potential_snarked_ledgers)	×
106
107	let enqueue_snarked_ledger ~location t =
108	Queue.enqueue t.potential_snarked_ledgers location ;	×
109	write_potential_snarked_ledgers_to_disk t	×
110
111	let dequeue_snarked_ledger t =
112	let location = Queue.dequeue_exn t.potential_snarked_ledgers in	×
113	Mina_stdlib_unix.File_system.rmrf location ;	×
114	write_potential_snarked_ledgers_to_disk t	×
115
116	let destroy t =
117	List.iter	×
118	(Queue.to_list t.potential_snarked_ledgers)	×
119	~f:Mina_stdlib_unix.File_system.rmrf ;
120	Mina_stdlib_unix.File_system.rmrf	×
121	(Locations.potential_snarked_ledgers t.factory.directory) ;	×
NEW 122	Ledger.Root.close t.snarked_ledger ;	×
123	t.factory.instance <- None	×
124
125	let close t =
NEW 126	Ledger.Root.close t.snarked_ledger ;	×
127	t.factory.instance <- None	×
128
129	let create factory =
130	let snarked_ledger =	×
131	Ledger.Root.create_single ~depth:factory.ledger_depth
132	~directory_name:(Locations.snarked_ledger factory.directory)	×
133	()
134	in
135	{ snarked_ledger; potential_snarked_ledgers = Queue.create (); factory }	×
136
137	(** When we load from disk,
138	1. Check the potential_snarked_ledgers to see if any one of these
139	matches the snarked_ledger_hash in persistent_frontier;
140	2. if none of those works, we load the old snarked_ledger and check if
141	the old snarked_ledger matches with persistent_frontier;
142	3. if not, we just reset all the persisted data and start from genesis
143	*)
144	let load_from_disk factory ~snarked_ledger_hash ~logger =
145	let potential_snarked_ledgers =	×
146	load_potential_snarked_ledgers_from_disk factory
147	in
148	let snarked_ledger =	×
149	List.fold_until potential_snarked_ledgers ~init:None
150	~f:(fun _ location ->
151	let potential_snarked_ledger =	×
152	Ledger.Root.create_single ~depth:factory.ledger_depth
153	~directory_name:location ()
154	in
155	let potential_snarked_ledger_hash =	×
156	Frozen_ledger_hash.of_ledger_hash
NEW 157	@@ Ledger.Root.merkle_root potential_snarked_ledger	×
158	in
159	[%log debug]	×
160	~metadata:
161	[ ( "potential_snarked_ledger_hash"
162	, Frozen_ledger_hash.to_yojson potential_snarked_ledger_hash )	×
163	]
164	"loaded potential_snarked_ledger from disk" ;
165	if	×
166	Frozen_ledger_hash.equal potential_snarked_ledger_hash
167	snarked_ledger_hash
168	then (	×
169	let snarked_ledger =
170	Ledger.Root.create_single ~depth:factory.ledger_depth
171	~directory_name:(Locations.tmp_snarked_ledger factory.directory)	×
172	()
173	in
174	match	×
175	transfer_snarked_root ~src:potential_snarked_ledger
176	~dest:snarked_ledger
177	with
178	\| Ok _ ->	×
179	Ledger.Root.close potential_snarked_ledger ;
180	Mina_stdlib_unix.File_system.rmrf	×
181	@@ Locations.snarked_ledger factory.directory ;	×
182	Sys.rename	×
183	(Locations.tmp_snarked_ledger factory.directory)	×
184	(Locations.snarked_ledger factory.directory) ;	×
185	List.iter potential_snarked_ledgers	×
186	~f:Mina_stdlib_unix.File_system.rmrf ;
187	Mina_stdlib_unix.File_system.rmrf	×
188	(Locations.potential_snarked_ledgers factory.directory) ;	×
189	Stop (Some snarked_ledger)	×
190	\| Error e ->	×
191	Ledger.Root.close potential_snarked_ledger ;
192	List.iter potential_snarked_ledgers	×
193	~f:Mina_stdlib_unix.File_system.rmrf ;
194	Mina_stdlib_unix.File_system.rmrf	×
195	(Locations.potential_snarked_ledgers factory.directory) ;	×
196	[%log' error factory.logger]	×
197	~metadata:[ ("error", `String (Error.to_string_hum e)) ]	×
198	"Ledger_transfer failed" ;
199	Stop None )	×
200	else (	×
201	Ledger.Root.close potential_snarked_ledger ;
202	Continue None ) )	×
203	~finish:(fun _ ->
204	List.iter potential_snarked_ledgers	×
205	~f:Mina_stdlib_unix.File_system.rmrf ;
206	Mina_stdlib_unix.File_system.rmrf	×
207	(Locations.potential_snarked_ledgers factory.directory) ;	×
208	None )	×
209	in
210	match snarked_ledger with	×
211	\| None ->	×
212	let snarked_ledger =
213	Ledger.Root.create_single ~depth:factory.ledger_depth
214	~directory_name:(Locations.snarked_ledger factory.directory)	×
215	()
216	in
217	let potential_snarked_ledger_hash =	×
218	Frozen_ledger_hash.of_ledger_hash
NEW 219	@@ Ledger.Root.merkle_root snarked_ledger	×
220	in
221	if	×
222	Frozen_ledger_hash.equal potential_snarked_ledger_hash
223	snarked_ledger_hash
224	then
225	Ok	×
226	{ snarked_ledger
227	; potential_snarked_ledgers = Queue.create ()	×
228	; factory
229	}
230	else (	×
231	Ledger.Root.close snarked_ledger ;
232	Error `Snarked_ledger_mismatch )	×
233	\| Some snarked_ledger ->	×
234	Ok
235	{ snarked_ledger
236	; potential_snarked_ledgers = Queue.create ()	×
237	; factory
238	}
239
UNCOV 240	let snarked_ledger { snarked_ledger; _ } = snarked_ledger	×
241
242	let set_root_identifier t new_root_identifier =
243	[%log' trace t.factory.logger]	×
244	~metadata:
245	[ ("root_identifier", Root_identifier.to_yojson new_root_identifier) ]	×
246	"Setting persistent root identifier" ;
247	let size = Root_identifier.Stable.Latest.bin_size_t new_root_identifier in	×
248	with_file (Locations.root_identifier t.factory.directory) `Write ~size	×
249	~f:(fun buf ->
250	ignore	×
251	( Root_identifier.Stable.Latest.bin_write_t buf ~pos:0	×
252	new_root_identifier
253	: int ) )
254
255	(* defaults to genesis *)
256	let load_root_identifier t =
257	let file = Locations.root_identifier t.factory.directory in	×
258	match Unix.access file [ `Exists; `Read ] with	×
259	\| Error _ ->	×
260	None
261	\| Ok () ->	×
262	with_file file `Read ~f:(fun buf ->
263	let root_identifier =	×
264	Root_identifier.Stable.Latest.bin_read_t buf ~pos_ref:(ref 0)
265	in
266	[%log' trace t.factory.logger]	×
267	~metadata:
268	[ ("root_identifier", Root_identifier.to_yojson root_identifier)	×
269	]
270	"Loaded persistent root identifier" ;
271	Some root_identifier )	×
272
273	let set_root_state_hash t state_hash = set_root_identifier t { state_hash }	×
274	end
275
276	type t = Factory_type.t
277
278	let create ~logger ~directory ~ledger_depth =
279	{ directory; logger; instance = None; ledger_depth }	×
280
281	let create_instance_exn t =
282	assert (Option.is_none t.instance) ;	×
283	let instance = Instance.create t in
284	t.instance <- Some instance ;	×
285	instance
286
287	let load_from_disk_exn t ~snarked_ledger_hash ~logger =
288	let open Result.Let_syntax in	×
289	assert (Option.is_none t.instance) ;	×
290	let%map instance = Instance.load_from_disk t ~snarked_ledger_hash ~logger in	×
291	t.instance <- Some instance ;	×
292	instance
293
294	let with_instance_exn t ~f =
295	let instance = create_instance_exn t in	×
296	let x = f instance in	×
297	Instance.close instance ; x	×
298
299	let reset_to_genesis_exn t ~precomputed_values =
300	assert (Option.is_none t.instance) ;	×
301	Mina_stdlib_unix.File_system.rmrf t.directory ;
302	with_instance_exn t ~f:(fun instance ->	×
303	ignore	×
NEW 304	( Precomputed_values.populate_root precomputed_values	×
NEW 305	(Instance.snarked_ledger instance)	×
NEW 306	\|> Or_error.map ~f:Ledger.Root.as_unmasked	×
307	: Ledger.Any_ledger.witness Or_error.t ) ;
308	Instance.set_root_identifier instance
309	(genesis_root_identifier
310	~genesis_state_hash:
311	(Precomputed_values.genesis_state_hashes precomputed_values)	×
312	.state_hash ) )	98✔

MinaProtocol / mina / 431

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous