2863

Committed 05 Nov 2024 06:20PM UTC coverage: 30.754% (-16.6%) from 47.311%

Build # 2863

Build Type

push

buildkite

Committed by

web-flow

Commit Message

Merge pull request #16296 from MinaProtocol/dkijania/more_multi_jobs

more multi jobs in CI

Run Details

20276 of 65930 relevant lines covered (30.75%)

8631.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.5

/src/lib/pickles/proof_cache.ml

open Core_kernel

[@@@warning "-4-27"]

module Yojson_map = Map.Make (struct
  type t =
    [ `Null
    | `Bool of bool
    | `Int of int
    | `Intlit of string
    | `Float of float
    | `String of string
    | `Assoc of (string * t) list
    | `List of t list
    | `Tuple of t list
    | `Variant of string * t option ]
  [@@deriving compare, sexp]
end)

[@@@warning "+4+27"]

type t = Yojson.Safe.t Yojson_map.t Yojson_map.t ref

(* We use a slightly more verbose format here, so that it's easy to debug.
   There are some overheads to handling this, but the amount of computation we
   save by caching the proofs is orders of magnitude higher, so it's not really
   an issue.
*)
let to_yojson t =
  `List
    (Map.fold ~init:[] !t ~f:(fun ~key ~data xs ->
         let proofs =
           Map.fold ~init:[] data ~f:(fun ~key ~data xs ->
               `Assoc [ ("public_input", key); ("proof", data) ] :: xs )
         in
         `Assoc [ ("verification_key", key); ("proofs", `List proofs) ] :: xs )
    )

(* This mirrors the format of [to_yojson], carefully ensuring that we can
   decode what we encode, and reporting an error when the format differs from
   what we expect.

   Note that, since this is a cache, it should always be possible to regenerate
   proofs for the cache by starting with the empty cache and calling
   [to_yojson] on the result.
*)
let of_yojson t =
  Result.try_with (fun () ->
      match t with
      | `List xs ->
          let for_vks =
            List.map xs ~f:(function
              | `Assoc [ ("verification_key", key); ("proofs", `List proofs) ]
                ->
                  let proofs =
                    List.map proofs ~f:(function
                      | `Assoc [ ("public_input", key); ("proof", data) ] ->
                          (key, data)
                      | _ ->
                          failwith
                            "Expected fields `public_input`, `proof` as a \
                             record in that order; received something \
                             different" )
                  in
                  (key, Yojson_map.of_alist_exn proofs)
              | _ ->
                  failwith
                    "Expected fields `verification_key`, `proofs` as a record \
                     in that order, where `proofs` is a list; received \
                     something different" )
          in
          ref (Yojson_map.of_alist_exn for_vks)
      | _ ->
          failwith "Expected a list, got something different" )
  |> Result.map_error ~f:Exn.to_string

(* Alias types with a [deriving to_yojson] annotation, so that we don't have to
   spell out the serialization explicitly.
*)
module Json = struct
  type 'f or_infinity = 'f Kimchi_types.or_infinity =
    | Infinity
    | Finite of ('f * 'f)
  [@@deriving to_yojson]

  type 'caml_g poly_comm = 'caml_g Kimchi_types.poly_comm =
    { unshifted : 'caml_g array; shifted : 'caml_g option }
  [@@deriving to_yojson]

  type lookup_patterns = Kimchi_types.lookup_patterns =
    { xor : bool; lookup : bool; range_check : bool; foreign_field_mul : bool }
  [@@deriving to_yojson]

  type lookup_features = Kimchi_types.lookup_features =
    { patterns : lookup_patterns
    ; joint_lookup_used : bool
    ; uses_runtime_tables : bool
    }
  [@@deriving to_yojson]

  type lookups_used = Kimchi_types.VerifierIndex.Lookup.lookups_used =
    | Single
    | Joint
  [@@deriving to_yojson]

  type lookup_info = Kimchi_types.VerifierIndex.Lookup.lookup_info =
    { max_per_row : int; max_joint_size : int; features : lookup_features }
  [@@deriving to_yojson]

  type 't lookup_selectors =
        't Kimchi_types.VerifierIndex.Lookup.lookup_selectors =
    { lookup : 't option
    ; xor : 't option
    ; range_check : 't option
    ; ffmul : 't option
    }
  [@@deriving to_yojson]

  type 'poly_comm lookup = 'poly_comm Kimchi_types.VerifierIndex.Lookup.t =
    { joint_lookup_used : bool
    ; lookup_table : 'poly_comm array
    ; lookup_selectors : 'poly_comm lookup_selectors
    ; table_ids : 'poly_comm option
    ; lookup_info : lookup_info
    ; runtime_tables_selector : 'poly_comm option
    }
  [@@deriving to_yojson]

  type 'fr domain = 'fr Kimchi_types.VerifierIndex.domain =
    { log_size_of_group : int; group_gen : 'fr }
  [@@deriving to_yojson]

  type 'poly_comm verification_evals =
        'poly_comm Kimchi_types.VerifierIndex.verification_evals =
    { sigma_comm : 'poly_comm array
    ; coefficients_comm : 'poly_comm array
    ; generic_comm : 'poly_comm
    ; psm_comm : 'poly_comm
    ; complete_add_comm : 'poly_comm
    ; mul_comm : 'poly_comm
    ; emul_comm : 'poly_comm
    ; endomul_scalar_comm : 'poly_comm
    ; xor_comm : 'poly_comm option [@default None]
    ; range_check0_comm : 'poly_comm option [@default None]
    ; range_check1_comm : 'poly_comm option [@default None]
    ; foreign_field_add_comm : 'poly_comm option [@default None]
    ; foreign_field_mul_comm : 'poly_comm option [@default None]
    ; rot_comm : 'poly_comm option [@default None]
    }
  [@@deriving to_yojson]

  type ('fr, 'srs, 'poly_comm) verifier_index =
        ('fr, 'srs, 'poly_comm) Kimchi_types.VerifierIndex.verifier_index =
    { domain : 'fr domain
    ; max_poly_size : int
    ; public : int
    ; prev_challenges : int
    ; srs : 'srs
    ; evals : 'poly_comm verification_evals
    ; shifts : 'fr array
    ; lookup_index : 'poly_comm lookup option
    ; zk_rows : int [@default 3]
    }
  [@@deriving to_yojson]

  let srs_to_yojson _ = `Null

  let step_verification_key_to_yojson =
    [%to_yojson:
      ( Backend.Tick.Field.t
      , srs
      , Backend.Tock.Field.t or_infinity poly_comm )
      verifier_index]

  let wrap_verification_key_to_yojson =
    [%to_yojson:
      ( Backend.Tock.Field.t
      , srs
      , Backend.Tick.Field.t or_infinity poly_comm )
      verifier_index]
end

let empty () = ref Yojson_map.empty

let get_proof t ~verification_key ~public_input =
  let open Option.Let_syntax in
  let%bind for_vk = Map.find !t verification_key in
  Map.find for_vk public_input

let get_step_proof t ~keypair ~public_input =
  let open Option.Let_syntax in
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fp.get public_input i )
    |> [%to_yojson: Backend.Tick.Field.t array]
  in
  let verification_key =
    Backend.Tick.Keypair.vk keypair |> Json.step_verification_key_to_yojson
  in
  let%bind proof_json = get_proof t ~verification_key ~public_input in
  Option.try_with (fun () ->
      Result.ok_or_failwith @@ Backend.Tick.Proof.of_yojson proof_json )

let get_wrap_proof t ~keypair ~public_input =
  let open Option.Let_syntax in
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fq.get public_input i )
    |> [%to_yojson: Backend.Tock.Field.t array]
  in
  let verification_key =
    Backend.Tock.Keypair.vk keypair |> Json.wrap_verification_key_to_yojson
  in
  let%bind proof_json = get_proof t ~verification_key ~public_input in
  Option.try_with (fun () ->
      Result.ok_or_failwith @@ Backend.Tock.Proof.of_yojson proof_json )

let set_proof t ~verification_key ~public_input proof =
  t :=
    Map.update !t verification_key ~f:(function
      | None ->
          Yojson_map.singleton public_input proof
      | Some for_vk ->
          Map.set for_vk ~key:public_input ~data:proof )

let set_step_proof t ~keypair ~public_input proof =
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fp.get public_input i )
    |> [%to_yojson: Backend.Tick.Field.t array]
  in
  let verification_key =
    Backend.Tick.Keypair.vk keypair |> Json.step_verification_key_to_yojson
  in
  let proof_json = Backend.Tick.Proof.to_yojson proof in
  set_proof t ~verification_key ~public_input proof_json

let set_wrap_proof t ~keypair ~public_input proof =
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fq.get public_input i )
    |> [%to_yojson: Backend.Tock.Field.t array]
  in
  let verification_key =
    Backend.Tock.Keypair.vk keypair |> Json.wrap_verification_key_to_yojson
  in
  let proof_json = Backend.Tock.Proof.to_yojson proof in
  set_proof t ~verification_key ~public_input proof_json

let is_env_var_set_requesting_error_for_proofs () =
  match Sys.getenv_opt "ERROR_ON_PROOF" with
  | Some ("true" | "t" (* insert whatever value is okay here *)) ->
      true
  | None | Some _ ->
      false

1	open Core_kernel	10✔
2
3	[@@@warning "-4-27"]
4
5	module Yojson_map = Map.Make (struct
6	type t =	×
7	[ `Null	×
8	\| `Bool of bool	×
9	\| `Int of int	×
10	\| `Intlit of string	×
11	\| `Float of float	×
12	\| `String of string	×
13	\| `Assoc of (string * t) list	×
14	\| `List of t list	×
15	\| `Tuple of t list	×
16	\| `Variant of string * t option ]	×
17	[@@deriving compare, sexp]	×
18	end)
19
20	[@@@warning "+4+27"]
21
22	type t = Yojson.Safe.t Yojson_map.t Yojson_map.t ref
23
24	(* We use a slightly more verbose format here, so that it's easy to debug.
25	There are some overheads to handling this, but the amount of computation we
26	save by caching the proofs is orders of magnitude higher, so it's not really
27	an issue.
28	*)
29	let to_yojson t =
30	`List	×
31	(Map.fold ~init:[] !t ~f:(fun ~key ~data xs ->	×
32	let proofs =	×
33	Map.fold ~init:[] data ~f:(fun ~key ~data xs ->
34	`Assoc [ ("public_input", key); ("proof", data) ] :: xs )	×
35	in
36	`Assoc [ ("verification_key", key); ("proofs", `List proofs) ] :: xs )	×
37	)
38
39	(* This mirrors the format of [to_yojson], carefully ensuring that we can
40	decode what we encode, and reporting an error when the format differs from
41	what we expect.
42
43	Note that, since this is a cache, it should always be possible to regenerate
44	proofs for the cache by starting with the empty cache and calling
45	[to_yojson] on the result.
46	*)
47	let of_yojson t =
48	Result.try_with (fun () ->	×
49	match t with	×
50	\| `List xs ->	×
51	let for_vks =
52	List.map xs ~f:(function
53	\| `Assoc [ ("verification_key", key); ("proofs", `List proofs) ]	×
54	->
55	let proofs =
56	List.map proofs ~f:(function
57	\| `Assoc [ ("public_input", key); ("proof", data) ] ->	×
58	(key, data)
59	\| _ ->	×
60	failwith
61	"Expected fields `public_input`, `proof` as a \
62	record in that order; received something \
63	different" )
64	in
65	(key, Yojson_map.of_alist_exn proofs)	×
66	\| _ ->	×
67	failwith
68	"Expected fields `verification_key`, `proofs` as a record \
69	in that order, where `proofs` is a list; received \
70	something different" )
71	in
72	ref (Yojson_map.of_alist_exn for_vks)	×
73	\| _ ->	×
74	failwith "Expected a list, got something different" )
75	\|> Result.map_error ~f:Exn.to_string
76
77	(* Alias types with a [deriving to_yojson] annotation, so that we don't have to
78	spell out the serialization explicitly.
79	*)
80	module Json = struct
81	type 'f or_infinity = 'f Kimchi_types.or_infinity =	×
82	\| Infinity
83	\| Finite of ('f * 'f)	×
84	[@@deriving to_yojson]
85
86	type 'caml_g poly_comm = 'caml_g Kimchi_types.poly_comm =	×
87	{ unshifted : 'caml_g array; shifted : 'caml_g option }	×
88	[@@deriving to_yojson]
89
90	type lookup_patterns = Kimchi_types.lookup_patterns =	×
91	{ xor : bool; lookup : bool; range_check : bool; foreign_field_mul : bool }	×
92	[@@deriving to_yojson]
93
94	type lookup_features = Kimchi_types.lookup_features =	×
95	{ patterns : lookup_patterns	×
96	; joint_lookup_used : bool	×
97	; uses_runtime_tables : bool	×
98	}
99	[@@deriving to_yojson]
100
101	type lookups_used = Kimchi_types.VerifierIndex.Lookup.lookups_used =	×
102	\| Single
103	\| Joint
104	[@@deriving to_yojson]
105
106	type lookup_info = Kimchi_types.VerifierIndex.Lookup.lookup_info =	×
107	{ max_per_row : int; max_joint_size : int; features : lookup_features }	×
108	[@@deriving to_yojson]
109
110	type 't lookup_selectors =	×
111	't Kimchi_types.VerifierIndex.Lookup.lookup_selectors =
112	{ lookup : 't option	×
113	; xor : 't option	×
114	; range_check : 't option	×
115	; ffmul : 't option	×
116	}
117	[@@deriving to_yojson]
118
119	type 'poly_comm lookup = 'poly_comm Kimchi_types.VerifierIndex.Lookup.t =	×
120	{ joint_lookup_used : bool	×
121	; lookup_table : 'poly_comm array	×
122	; lookup_selectors : 'poly_comm lookup_selectors	×
123	; table_ids : 'poly_comm option	×
124	; lookup_info : lookup_info	×
125	; runtime_tables_selector : 'poly_comm option	×
126	}
127	[@@deriving to_yojson]	×
128
129	type 'fr domain = 'fr Kimchi_types.VerifierIndex.domain =	×
130	{ log_size_of_group : int; group_gen : 'fr }	×
131	[@@deriving to_yojson]
132
133	type 'poly_comm verification_evals =	×
134	'poly_comm Kimchi_types.VerifierIndex.verification_evals =
135	{ sigma_comm : 'poly_comm array	×
136	; coefficients_comm : 'poly_comm array	×
137	; generic_comm : 'poly_comm	×
138	; psm_comm : 'poly_comm	×
139	; complete_add_comm : 'poly_comm	×
140	; mul_comm : 'poly_comm	×
141	; emul_comm : 'poly_comm	×
142	; endomul_scalar_comm : 'poly_comm	×
143	; xor_comm : 'poly_comm option [@default None]	×
144	; range_check0_comm : 'poly_comm option [@default None]	×
145	; range_check1_comm : 'poly_comm option [@default None]	×
146	; foreign_field_add_comm : 'poly_comm option [@default None]	×
147	; foreign_field_mul_comm : 'poly_comm option [@default None]	×
148	; rot_comm : 'poly_comm option [@default None]	×
149	}
150	[@@deriving to_yojson]
151
152	type ('fr, 'srs, 'poly_comm) verifier_index =	×
153	('fr, 'srs, 'poly_comm) Kimchi_types.VerifierIndex.verifier_index =
154	{ domain : 'fr domain	×
155	; max_poly_size : int	×
156	; public : int	×
157	; prev_challenges : int	×
158	; srs : 'srs	×
159	; evals : 'poly_comm verification_evals	×
160	; shifts : 'fr array	×
161	; lookup_index : 'poly_comm lookup option	×
162	; zk_rows : int [@default 3]	×
163	}
164	[@@deriving to_yojson]	×
165
166	let srs_to_yojson _ = `Null	×
167
168	let step_verification_key_to_yojson =
169	[%to_yojson:
170	( Backend.Tick.Field.t	×
171	, srs	×
172	, Backend.Tock.Field.t or_infinity poly_comm )	×
173	verifier_index]	×
174
175	let wrap_verification_key_to_yojson =
176	[%to_yojson:
177	( Backend.Tock.Field.t	×
178	, srs	×
179	, Backend.Tick.Field.t or_infinity poly_comm )	×
180	verifier_index]	×
181	end
182
183	let empty () = ref Yojson_map.empty	×
184
185	let get_proof t ~verification_key ~public_input =
186	let open Option.Let_syntax in	×
187	let%bind for_vk = Map.find !t verification_key in	×
188	Map.find for_vk public_input	×
189
190	let get_step_proof t ~keypair ~public_input =
191	let open Option.Let_syntax in	×
192	let public_input =
193	let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
194	Array.init len ~f:(fun i ->	×
195	Kimchi_bindings.FieldVectors.Fp.get public_input i )	×
196	\|> [%to_yojson: Backend.Tick.Field.t array]	×
197	in
198	let verification_key =
199	Backend.Tick.Keypair.vk keypair \|> Json.step_verification_key_to_yojson	×
200	in
201	let%bind proof_json = get_proof t ~verification_key ~public_input in	×
202	Option.try_with (fun () ->	×
203	Result.ok_or_failwith @@ Backend.Tick.Proof.of_yojson proof_json )	×
204
205	let get_wrap_proof t ~keypair ~public_input =
206	let open Option.Let_syntax in	×
207	let public_input =
208	let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
209	Array.init len ~f:(fun i ->	×
210	Kimchi_bindings.FieldVectors.Fq.get public_input i )	×
211	\|> [%to_yojson: Backend.Tock.Field.t array]	×
212	in
213	let verification_key =
214	Backend.Tock.Keypair.vk keypair \|> Json.wrap_verification_key_to_yojson	×
215	in
216	let%bind proof_json = get_proof t ~verification_key ~public_input in	×
217	Option.try_with (fun () ->	×
218	Result.ok_or_failwith @@ Backend.Tock.Proof.of_yojson proof_json )	×
219
220	let set_proof t ~verification_key ~public_input proof =
221	t :=	×
222	Map.update !t verification_key ~f:(function	×
223	\| None ->	×
224	Yojson_map.singleton public_input proof
225	\| Some for_vk ->	×
226	Map.set for_vk ~key:public_input ~data:proof )
227
228	let set_step_proof t ~keypair ~public_input proof =
229	let public_input =	×
230	let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
231	Array.init len ~f:(fun i ->	×
232	Kimchi_bindings.FieldVectors.Fp.get public_input i )	×
233	\|> [%to_yojson: Backend.Tick.Field.t array]	×
234	in
235	let verification_key =
236	Backend.Tick.Keypair.vk keypair \|> Json.step_verification_key_to_yojson	×
237	in
238	let proof_json = Backend.Tick.Proof.to_yojson proof in	×
239	set_proof t ~verification_key ~public_input proof_json	×
240
241	let set_wrap_proof t ~keypair ~public_input proof =
242	let public_input =	×
243	let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
244	Array.init len ~f:(fun i ->	×
245	Kimchi_bindings.FieldVectors.Fq.get public_input i )	×
246	\|> [%to_yojson: Backend.Tock.Field.t array]	×
247	in
248	let verification_key =
249	Backend.Tock.Keypair.vk keypair \|> Json.wrap_verification_key_to_yojson	×
250	in
251	let proof_json = Backend.Tock.Proof.to_yojson proof in	×
252	set_proof t ~verification_key ~public_input proof_json	×
253
254	let is_env_var_set_requesting_error_for_proofs () =
255	match Sys.getenv_opt "ERROR_ON_PROOF" with	×
256	\| Some ("true" \| "t" (* insert whatever value is okay here *)) ->	×
257	true
258	\| None \| Some _ ->	×
259	false	20✔

MinaProtocol / mina / 2863

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous