2863

Committed 05 Nov 2024 06:20PM UTC coverage: 30.754% (-16.6%) from 47.311%

Build # 2863

Build Type

push

buildkite

Committed by

web-flow

Commit Message

Merge pull request #16296 from MinaProtocol/dkijania/more_multi_jobs

more multi jobs in CI

Run Details

20276 of 65930 relevant lines covered (30.75%)

8631.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.04

/src/lib/pickles/cache.ml

open Core_kernel

module Step = struct
  module Key = struct
    module Proving = struct
      type t =
        Type_equal.Id.Uid.t
        * Snark_keys_header.t
        * int
        * Backend.Tick.R1CS_constraint_system.t

      let to_string : t -> _ = function
        | _id, header, n, _h ->
            sprintf !"step-%s-%s-%d-%s" header.kind.type_ header.kind.identifier
              n header.identifying_hash
    end

    module Verification = struct
      type t = Type_equal.Id.Uid.t * Snark_keys_header.t * int * Md5.t
      [@@deriving sexp]

      let to_string : t -> _ = function
        | _id, header, n, _h ->
            sprintf !"vk-step-%s-%s-%d-%s" header.kind.type_
              header.kind.identifier n header.identifying_hash
    end
    [@@warning "-4"]
  end

  type storable =
    (Key.Proving.t, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t

  type vk_storable =
    ( Key.Verification.t
    , Kimchi_bindings.Protocol.VerifierIndex.Fp.t )
    Key_cache.Sync.Disk_storable.t

  let storable =
    Key_cache.Sync.Disk_storable.simple Key.Proving.to_string
      (fun (_, header, _, cs) ~path ->
        Or_error.try_with_join (fun () ->
            let open Or_error.Let_syntax in
            let%map header_read, index =
              Snark_keys_header.read_with_header
                ~read_data:(fun ~offset ->
                  Kimchi_bindings.Protocol.Index.Fp.read (Some offset)
                    (Backend.Tick.Keypair.load_urs ()) )
                path
            in
            [%test_eq: int] header.header_version header_read.header_version ;
            [%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;
            [%test_eq: Snark_keys_header.Constraint_constants.t]
              header.constraint_constants header_read.constraint_constants ;
            [%test_eq: string] header.constraint_system_hash
              header_read.constraint_system_hash ;
            { Backend.Tick.Keypair.index; cs } ) )
      (fun (_, header, _, _) t path ->
        Or_error.try_with (fun () ->
            Snark_keys_header.write_with_header
              ~expected_max_size_log2:33 (* 8 GB should be enough *)
              ~append_data:
                (Kimchi_bindings.Protocol.Index.Fp.write (Some true)
                   t.Backend.Tick.Keypair.index )
              header path ) )

  let vk_storable =
    Key_cache.Sync.Disk_storable.simple Key.Verification.to_string
      (fun (_, header, _, _) ~path ->
        Or_error.try_with_join (fun () ->
            let open Or_error.Let_syntax in
            let%map header_read, index =
              Snark_keys_header.read_with_header
                ~read_data:(fun ~offset path ->
                  Kimchi_bindings.Protocol.VerifierIndex.Fp.read (Some offset)
                    (Backend.Tick.Keypair.load_urs ())
                    path )
                path
            in
            [%test_eq: int] header.header_version header_read.header_version ;
            [%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;
            [%test_eq: Snark_keys_header.Constraint_constants.t]
              header.constraint_constants header_read.constraint_constants ;
            [%test_eq: string] header.constraint_system_hash
              header_read.constraint_system_hash ;
            index ) )
      (fun (_, header, _, _) x path ->
        Or_error.try_with (fun () ->
            Snark_keys_header.write_with_header
              ~expected_max_size_log2:33 (* 8 GB should be enough *)
              ~append_data:
                (Kimchi_bindings.Protocol.VerifierIndex.Fp.write (Some true) x)
              header path ) )

  let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p
      ?(s_v = vk_storable) k_v =
    let open Impls.Step in
    let pk =
      lazy
        (let%map.Promise k_p = Lazy.force k_p in
         match
           Common.time "step keypair read" (fun () ->
               Key_cache.Sync.read cache s_p k_p )
         with
         | Ok (pk, dirty) ->
             Common.time "step keypair create" (fun () -> (pk, dirty))
         | Error _e ->
             let _, _, _, sys = k_p in
             let r =
               Common.time "stepkeygen" (fun () ->
                   Keypair.generate ~prev_challenges sys )
             in
             Timer.clock __LOC__ ;
             ignore
               ( Key_cache.Sync.write cache s_p k_p (Keypair.pk r)
                 : unit Or_error.t ) ;
             (Keypair.pk r, `Generated_something) )
    in
    let vk =
      lazy
        (let%bind.Promise k_v = Lazy.force k_v in
         match
           Common.time "step vk read" (fun () ->
               Key_cache.Sync.read cache s_v k_v )
         with
         | Ok (vk, _) ->
             Promise.return (vk, `Cache_hit)
         | Error _e ->
             let%map.Promise pk, c = Lazy.force pk in
             let vk = Backend.Tick.Keypair.vk pk in
             ignore (Key_cache.Sync.write cache s_v k_v vk : unit Or_error.t) ;
             (vk, c) )
    in
    (pk, vk)
end

module Wrap = struct
  module Key = struct
    module Verification = struct
      type t = Type_equal.Id.Uid.t * Snark_keys_header.t * Md5.t
      [@@deriving sexp]

      let equal ((_, x1, y1) : t) ((_, x2, y2) : t) =
        [%equal: unit * Md5.t] ((* TODO: *) ignore x1, y1) (ignore x2, y2)

      let to_string : t -> _ = function
        | _id, header, _h ->
            sprintf !"vk-wrap-%s-%s-%s" header.kind.type_ header.kind.identifier
              header.identifying_hash
    end
    [@@warning "-4"]

    module Proving = struct
      type t =
        Type_equal.Id.Uid.t
        * Snark_keys_header.t
        * Backend.Tock.R1CS_constraint_system.t

      let to_string : t -> _ = function
        | _id, header, _h ->
            sprintf !"wrap-%s-%s-%s" header.kind.type_ header.kind.identifier
              header.identifying_hash
    end
  end

  type storable =
    (Key.Proving.t, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t

  type vk_storable =
    (Key.Verification.t, Verification_key.t) Key_cache.Sync.Disk_storable.t

  let storable =
    Key_cache.Sync.Disk_storable.simple Key.Proving.to_string
      (fun (_, header, cs) ~path ->
        Or_error.try_with_join (fun () ->
            let open Or_error.Let_syntax in
            let%map header_read, index =
              Snark_keys_header.read_with_header
                ~read_data:(fun ~offset ->
                  Kimchi_bindings.Protocol.Index.Fq.read (Some offset)
                    (Backend.Tock.Keypair.load_urs ()) )
                path
            in
            [%test_eq: int] header.header_version header_read.header_version ;
            [%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;
            [%test_eq: Snark_keys_header.Constraint_constants.t]
              header.constraint_constants header_read.constraint_constants ;
            [%test_eq: string] header.constraint_system_hash
              header_read.constraint_system_hash ;
            { Backend.Tock.Keypair.index; cs } ) )
      (fun (_, header, _) t path ->
        Or_error.try_with (fun () ->
            Snark_keys_header.write_with_header
              ~expected_max_size_log2:33 (* 8 GB should be enough *)
              ~append_data:
                (Kimchi_bindings.Protocol.Index.Fq.write (Some true) t.index)
              header path ) )

  let vk_storable =
    Key_cache.Sync.Disk_storable.simple Key.Verification.to_string
      (fun (_, header, _cs) ~path ->
        Or_error.try_with_join (fun () ->
            let open Or_error.Let_syntax in
            let%map header_read, index =
              Snark_keys_header.read_with_header
                ~read_data:(fun ~offset:_ path ->
                  Binable.of_string
                    (module Verification_key.Stable.Latest)
                    (In_channel.read_all path) )
                path
            in
            [%test_eq: int] header.header_version header_read.header_version ;
            [%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;
            [%test_eq: Snark_keys_header.Constraint_constants.t]
              header.constraint_constants header_read.constraint_constants ;
            [%test_eq: string] header.constraint_system_hash
              header_read.constraint_system_hash ;
            index ) )
      (fun (_, header, _) t path ->
        Or_error.try_with (fun () ->
            Snark_keys_header.write_with_header
              ~expected_max_size_log2:33 (* 8 GB should be enough *)
              ~append_data:(fun path ->
                Out_channel.with_file ~append:true path ~f:(fun file ->
                    Out_channel.output_string file
                      (Binable.to_string
                         (module Verification_key.Stable.Latest)
                         t ) ) )
              header path ) )

  let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p
      ?(s_v = vk_storable) k_v =
    let module Vk = Verification_key in
    let open Impls.Wrap in
    let pk =
      lazy
        (let%map.Promise k = Lazy.force k_p in
         match
           Common.time "wrap key read" (fun () ->
               Key_cache.Sync.read cache s_p k )
         with
         | Ok (pk, d) ->
             (pk, d)
         | Error _e ->
             let _, _, sys = k in
             let r =
               Common.time "wrapkeygen" (fun () ->
                   Keypair.generate ~prev_challenges sys )
             in
             ignore
               ( Key_cache.Sync.write cache s_p k (Keypair.pk r)
                 : unit Or_error.t ) ;
             (Keypair.pk r, `Generated_something) )
    in
    let vk =
      lazy
        (let%bind.Promise k_v = Lazy.force k_v in
         match Key_cache.Sync.read cache s_v k_v with
         | Ok (vk, d) ->
             Promise.return (vk, d)
         | Error _e ->
             let%map.Promise pk, _dirty = Lazy.force pk in
             let vk = Backend.Tock.Keypair.vk pk in
             let vk : Vk.t =
               { index = vk
               ; commitments =
                   Kimchi_pasta.Pallas_based_plonk.Keypair.vk_commitments vk
               ; data =
                   (let open Kimchi_bindings.Protocol.Index.Fq in
                   { constraints = domain_d1_size pk.index })
               }
             in
             ignore (Key_cache.Sync.write cache s_v k_v vk : unit Or_error.t) ;
             let _vk = Key_cache.Sync.read cache s_v k_v in
             (vk, `Generated_something) )
    in
    (pk, vk)
end

1	open Core_kernel	10✔
2
3	module Step = struct
4	module Key = struct
5	module Proving = struct
6	type t =
7	Type_equal.Id.Uid.t
8	* Snark_keys_header.t
9	* int
10	* Backend.Tick.R1CS_constraint_system.t
11
12	let to_string : t -> _ = function
13	\| _id, header, n, _h ->	25✔
14	sprintf !"step-%s-%s-%d-%s" header.kind.type_ header.kind.identifier
15	n header.identifying_hash
16	end
17
18	module Verification = struct
19	type t = Type_equal.Id.Uid.t * Snark_keys_header.t * int * Md5.t	×
20	[@@deriving sexp]
21
22	let to_string : t -> _ = function
23	\| _id, header, n, _h ->	25✔
24	sprintf !"vk-step-%s-%s-%d-%s" header.kind.type_
25	header.kind.identifier n header.identifying_hash
26	end
27	[@@warning "-4"]
28	end
29
30	type storable =
31	(Key.Proving.t, Backend.Tick.Keypair.t) Key_cache.Sync.Disk_storable.t
32
33	type vk_storable =
34	( Key.Verification.t
35	, Kimchi_bindings.Protocol.VerifierIndex.Fp.t )
36	Key_cache.Sync.Disk_storable.t
37
38	let storable =
39	Key_cache.Sync.Disk_storable.simple Key.Proving.to_string	10✔
40	(fun (_, header, _, cs) ~path ->
41	Or_error.try_with_join (fun () ->	×
42	let open Or_error.Let_syntax in	×
43	let%map header_read, index =
44	Snark_keys_header.read_with_header	×
45	~read_data:(fun ~offset ->
46	Kimchi_bindings.Protocol.Index.Fp.read (Some offset)	×
47	(Backend.Tick.Keypair.load_urs ()) )	×
48	path
49	in
50	[%test_eq: int] header.header_version header_read.header_version ;	×
51	[%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;	×
52	[%test_eq: Snark_keys_header.Constraint_constants.t]	×
53	header.constraint_constants header_read.constraint_constants ;
54	[%test_eq: string] header.constraint_system_hash	×
55	header_read.constraint_system_hash ;
56	{ Backend.Tick.Keypair.index; cs } ) )	×
57	(fun (_, header, _, _) t path ->
58	Or_error.try_with (fun () ->	5✔
59	Snark_keys_header.write_with_header	5✔
60	~expected_max_size_log2:33 (* 8 GB should be enough *)
61	~append_data:
62	(Kimchi_bindings.Protocol.Index.Fp.write (Some true)	5✔
63	t.Backend.Tick.Keypair.index )
64	header path ) )
65
66	let vk_storable =
67	Key_cache.Sync.Disk_storable.simple Key.Verification.to_string	10✔
68	(fun (_, header, _, _) ~path ->
69	Or_error.try_with_join (fun () ->	×
70	let open Or_error.Let_syntax in	×
71	let%map header_read, index =
72	Snark_keys_header.read_with_header	×
73	~read_data:(fun ~offset path ->
74	Kimchi_bindings.Protocol.VerifierIndex.Fp.read (Some offset)	×
75	(Backend.Tick.Keypair.load_urs ())	×
76	path )
77	path
78	in
79	[%test_eq: int] header.header_version header_read.header_version ;	×
80	[%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;	×
81	[%test_eq: Snark_keys_header.Constraint_constants.t]	×
82	header.constraint_constants header_read.constraint_constants ;
83	[%test_eq: string] header.constraint_system_hash	×
84	header_read.constraint_system_hash ;
85	index ) )	×
86	(fun (_, header, _, _) x path ->
87	Or_error.try_with (fun () ->	5✔
88	Snark_keys_header.write_with_header	5✔
89	~expected_max_size_log2:33 (* 8 GB should be enough *)
90	~append_data:
91	(Kimchi_bindings.Protocol.VerifierIndex.Fp.write (Some true) x)	5✔
92	header path ) )
93
94	let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p	×
95	?(s_v = vk_storable) k_v =	×
96	let open Impls.Step in	24✔
97	let pk =
98	lazy
99	(let%map.Promise k_p = Lazy.force k_p in	23✔
100	match	23✔
101	Common.time "step keypair read" (fun () ->
102	Key_cache.Sync.read cache s_p k_p )	23✔
103	with
104	\| Ok (pk, dirty) ->	×
105	Common.time "step keypair create" (fun () -> (pk, dirty))	×
106	\| Error _e ->	23✔
107	let _, _, _, sys = k_p in
108	let r =
109	Common.time "stepkeygen" (fun () ->
110	Keypair.generate ~prev_challenges sys )	23✔
111	in
112	Timer.clock __LOC__ ;	23✔
113	ignore	23✔
114	( Key_cache.Sync.write cache s_p k_p (Keypair.pk r)	23✔
115	: unit Or_error.t ) ;
116	(Keypair.pk r, `Generated_something) )	23✔
117	in
118	let vk =
119	lazy
120	(let%bind.Promise k_v = Lazy.force k_v in	23✔
121	match	23✔
122	Common.time "step vk read" (fun () ->
123	Key_cache.Sync.read cache s_v k_v )	23✔
124	with
125	\| Ok (vk, _) ->	×
126	Promise.return (vk, `Cache_hit)
127	\| Error _e ->	23✔
128	let%map.Promise pk, c = Lazy.force pk in	23✔
129	let vk = Backend.Tick.Keypair.vk pk in	23✔
130	ignore (Key_cache.Sync.write cache s_v k_v vk : unit Or_error.t) ;	23✔
131	(vk, c) )
132	in
133	(pk, vk)
134	end
135
136	module Wrap = struct
137	module Key = struct
138	module Verification = struct
139	type t = Type_equal.Id.Uid.t * Snark_keys_header.t * Md5.t	×
140	[@@deriving sexp]
141
142	let equal ((_, x1, y1) : t) ((_, x2, y2) : t) =
143	[%equal: unit * Md5.t] ((* TODO: *) ignore x1, y1) (ignore x2, y2)	×
144
145	let to_string : t -> _ = function
146	\| _id, header, _h ->	9✔
147	sprintf !"vk-wrap-%s-%s-%s" header.kind.type_ header.kind.identifier
148	header.identifying_hash
149	end
150	[@@warning "-4"]
151
152	module Proving = struct
153	type t =
154	Type_equal.Id.Uid.t
155	* Snark_keys_header.t
156	* Backend.Tock.R1CS_constraint_system.t
157
158	let to_string : t -> _ = function
159	\| _id, header, _h ->	5✔
160	sprintf !"wrap-%s-%s-%s" header.kind.type_ header.kind.identifier
161	header.identifying_hash
162	end
163	end
164
165	type storable =
166	(Key.Proving.t, Backend.Tock.Keypair.t) Key_cache.Sync.Disk_storable.t
167
168	type vk_storable =
169	(Key.Verification.t, Verification_key.t) Key_cache.Sync.Disk_storable.t
170
171	let storable =
172	Key_cache.Sync.Disk_storable.simple Key.Proving.to_string	10✔
173	(fun (_, header, cs) ~path ->
174	Or_error.try_with_join (fun () ->	×
175	let open Or_error.Let_syntax in	×
176	let%map header_read, index =
177	Snark_keys_header.read_with_header	×
178	~read_data:(fun ~offset ->
179	Kimchi_bindings.Protocol.Index.Fq.read (Some offset)	×
180	(Backend.Tock.Keypair.load_urs ()) )	×
181	path
182	in
183	[%test_eq: int] header.header_version header_read.header_version ;	×
184	[%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;	×
185	[%test_eq: Snark_keys_header.Constraint_constants.t]	×
186	header.constraint_constants header_read.constraint_constants ;
187	[%test_eq: string] header.constraint_system_hash	×
188	header_read.constraint_system_hash ;
189	{ Backend.Tock.Keypair.index; cs } ) )	×
190	(fun (_, header, _) t path ->
191	Or_error.try_with (fun () ->	1✔
192	Snark_keys_header.write_with_header	1✔
193	~expected_max_size_log2:33 (* 8 GB should be enough *)
194	~append_data:
195	(Kimchi_bindings.Protocol.Index.Fq.write (Some true) t.index)	1✔
196	header path ) )
197
198	let vk_storable =
199	Key_cache.Sync.Disk_storable.simple Key.Verification.to_string	10✔
200	(fun (_, header, _cs) ~path ->
201	Or_error.try_with_join (fun () ->	1✔
202	let open Or_error.Let_syntax in	1✔
203	let%map header_read, index =
204	Snark_keys_header.read_with_header	1✔
205	~read_data:(fun ~offset:_ path ->
206	Binable.of_string	1✔
207	(module Verification_key.Stable.Latest)
208	(In_channel.read_all path) )	1✔
209	path
210	in
211	[%test_eq: int] header.header_version header_read.header_version ;	×
212	[%test_eq: Snark_keys_header.Kind.t] header.kind header_read.kind ;	×
213	[%test_eq: Snark_keys_header.Constraint_constants.t]	×
214	header.constraint_constants header_read.constraint_constants ;
215	[%test_eq: string] header.constraint_system_hash	×
216	header_read.constraint_system_hash ;
217	index ) )	×
218	(fun (_, header, _) t path ->
219	Or_error.try_with (fun () ->	1✔
220	Snark_keys_header.write_with_header	1✔
221	~expected_max_size_log2:33 (* 8 GB should be enough *)
222	~append_data:(fun path ->
223	Out_channel.with_file ~append:true path ~f:(fun file ->	1✔
224	Out_channel.output_string file	1✔
225	(Binable.to_string	1✔
226	(module Verification_key.Stable.Latest)
227	t ) ) )
228	header path ) )
229
230	let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p	×
231	?(s_v = vk_storable) k_v =	×
232	let module Vk = Verification_key in	8✔
233	let open Impls.Wrap in
234	let pk =
235	lazy
236	(let%map.Promise k = Lazy.force k_p in	7✔
237	match	7✔
238	Common.time "wrap key read" (fun () ->
239	Key_cache.Sync.read cache s_p k )	7✔
240	with
241	\| Ok (pk, d) ->	×
242	(pk, d)
243	\| Error _e ->	7✔
244	let _, _, sys = k in
245	let r =
246	Common.time "wrapkeygen" (fun () ->
247	Keypair.generate ~prev_challenges sys )	7✔
248	in
249	ignore	7✔
250	( Key_cache.Sync.write cache s_p k (Keypair.pk r)	7✔
251	: unit Or_error.t ) ;
252	(Keypair.pk r, `Generated_something) )	7✔
253	in
254	let vk =
255	lazy
256	(let%bind.Promise k_v = Lazy.force k_v in	7✔
257	match Key_cache.Sync.read cache s_v k_v with	7✔
258	\| Ok (vk, d) ->	×
259	Promise.return (vk, d)
260	\| Error _e ->	7✔
261	let%map.Promise pk, _dirty = Lazy.force pk in	7✔
262	let vk = Backend.Tock.Keypair.vk pk in	7✔
263	let vk : Vk.t =	7✔
264	{ index = vk
265	; commitments =
266	Kimchi_pasta.Pallas_based_plonk.Keypair.vk_commitments vk	7✔
267	; data =
268	(let open Kimchi_bindings.Protocol.Index.Fq in
269	{ constraints = domain_d1_size pk.index })	7✔
270	}
271	in
272	ignore (Key_cache.Sync.write cache s_v k_v vk : unit Or_error.t) ;	7✔
273	let _vk = Key_cache.Sync.read cache s_v k_v in
274	(vk, `Generated_something) )	7✔
275	in
276	(pk, vk)
277	end	20✔

MinaProtocol / mina / 2863

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous