2903

Committed 15 Nov 2024 01:59PM UTC coverage: 36.723% (-25.0%) from 61.682%

Build # 2903

Build Type

Pull #16342

buildkite

Committed by

dkijania

Commit Message

Merge branch 'dkijania/remove_publish_job_from_pr_comp' into dkijania/remove_publish_job_from_pr_dev

Pull Request Pull Request #16342: [DEV] Publish debians only on nightly and stable

Run Details

15 of 40 new or added lines in 14 files covered. (37.5%)

15175 existing lines in 340 files now uncovered.

24554 of 66863 relevant lines covered (36.72%)

20704.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.5

/src/lib/pickles/proof_cache.ml

open Core_kernel

[@@@warning "-4-27"]

module Yojson_map = Map.Make (struct
  type t =
    [ `Null
    | `Bool of bool
    | `Int of int
    | `Intlit of string
    | `Float of float
    | `String of string
    | `Assoc of (string * t) list
    | `List of t list
    | `Tuple of t list
    | `Variant of string * t option ]
  [@@deriving compare, sexp]
end)

[@@@warning "+4+27"]

type t = Yojson.Safe.t Yojson_map.t Yojson_map.t ref

(* We use a slightly more verbose format here, so that it's easy to debug.
   There are some overheads to handling this, but the amount of computation we
   save by caching the proofs is orders of magnitude higher, so it's not really
   an issue.
*)
let to_yojson t =
  `List
    (Map.fold ~init:[] !t ~f:(fun ~key ~data xs ->
         let proofs =
           Map.fold ~init:[] data ~f:(fun ~key ~data xs ->
               `Assoc [ ("public_input", key); ("proof", data) ] :: xs )
         in
         `Assoc [ ("verification_key", key); ("proofs", `List proofs) ] :: xs )
    )

(* This mirrors the format of [to_yojson], carefully ensuring that we can
   decode what we encode, and reporting an error when the format differs from
   what we expect.

   Note that, since this is a cache, it should always be possible to regenerate
   proofs for the cache by starting with the empty cache and calling
   [to_yojson] on the result.
*)
let of_yojson t =
  Result.try_with (fun () ->
      match t with
      | `List xs ->
          let for_vks =
            List.map xs ~f:(function
              | `Assoc [ ("verification_key", key); ("proofs", `List proofs) ]
                ->
                  let proofs =
                    List.map proofs ~f:(function
                      | `Assoc [ ("public_input", key); ("proof", data) ] ->
                          (key, data)
                      | _ ->
                          failwith
                            "Expected fields `public_input`, `proof` as a \
                             record in that order; received something \
                             different" )
                  in
                  (key, Yojson_map.of_alist_exn proofs)
              | _ ->
                  failwith
                    "Expected fields `verification_key`, `proofs` as a record \
                     in that order, where `proofs` is a list; received \
                     something different" )
          in
          ref (Yojson_map.of_alist_exn for_vks)
      | _ ->
          failwith "Expected a list, got something different" )
  |> Result.map_error ~f:Exn.to_string

(* Alias types with a [deriving to_yojson] annotation, so that we don't have to
   spell out the serialization explicitly.
*)
module Json = struct
  type 'f or_infinity = 'f Kimchi_types.or_infinity =
    | Infinity
    | Finite of ('f * 'f)
  [@@deriving to_yojson]

  type 'caml_g poly_comm = 'caml_g Kimchi_types.poly_comm =
    { unshifted : 'caml_g array; shifted : 'caml_g option }
  [@@deriving to_yojson]

  type lookup_patterns = Kimchi_types.lookup_patterns =
    { xor : bool; lookup : bool; range_check : bool; foreign_field_mul : bool }
  [@@deriving to_yojson]

  type lookup_features = Kimchi_types.lookup_features =
    { patterns : lookup_patterns
    ; joint_lookup_used : bool
    ; uses_runtime_tables : bool
    }
  [@@deriving to_yojson]

  type lookups_used = Kimchi_types.VerifierIndex.Lookup.lookups_used =
    | Single
    | Joint
  [@@deriving to_yojson]

  type lookup_info = Kimchi_types.VerifierIndex.Lookup.lookup_info =
    { max_per_row : int; max_joint_size : int; features : lookup_features }
  [@@deriving to_yojson]

  type 't lookup_selectors =
        't Kimchi_types.VerifierIndex.Lookup.lookup_selectors =
    { lookup : 't option
    ; xor : 't option
    ; range_check : 't option
    ; ffmul : 't option
    }
  [@@deriving to_yojson]

  type 'poly_comm lookup = 'poly_comm Kimchi_types.VerifierIndex.Lookup.t =
    { joint_lookup_used : bool
    ; lookup_table : 'poly_comm array
    ; lookup_selectors : 'poly_comm lookup_selectors
    ; table_ids : 'poly_comm option
    ; lookup_info : lookup_info
    ; runtime_tables_selector : 'poly_comm option
    }
  [@@deriving to_yojson]

  type 'fr domain = 'fr Kimchi_types.VerifierIndex.domain =
    { log_size_of_group : int; group_gen : 'fr }
  [@@deriving to_yojson]

  type 'poly_comm verification_evals =
        'poly_comm Kimchi_types.VerifierIndex.verification_evals =
    { sigma_comm : 'poly_comm array
    ; coefficients_comm : 'poly_comm array
    ; generic_comm : 'poly_comm
    ; psm_comm : 'poly_comm
    ; complete_add_comm : 'poly_comm
    ; mul_comm : 'poly_comm
    ; emul_comm : 'poly_comm
    ; endomul_scalar_comm : 'poly_comm
    ; xor_comm : 'poly_comm option [@default None]
    ; range_check0_comm : 'poly_comm option [@default None]
    ; range_check1_comm : 'poly_comm option [@default None]
    ; foreign_field_add_comm : 'poly_comm option [@default None]
    ; foreign_field_mul_comm : 'poly_comm option [@default None]
    ; rot_comm : 'poly_comm option [@default None]
    }
  [@@deriving to_yojson]

  type ('fr, 'srs, 'poly_comm) verifier_index =
        ('fr, 'srs, 'poly_comm) Kimchi_types.VerifierIndex.verifier_index =
    { domain : 'fr domain
    ; max_poly_size : int
    ; public : int
    ; prev_challenges : int
    ; srs : 'srs
    ; evals : 'poly_comm verification_evals
    ; shifts : 'fr array
    ; lookup_index : 'poly_comm lookup option
    ; zk_rows : int [@default 3]
    }
  [@@deriving to_yojson]

  let srs_to_yojson _ = `Null

  let step_verification_key_to_yojson =
    [%to_yojson:
      ( Backend.Tick.Field.t
      , srs
      , Backend.Tock.Field.t or_infinity poly_comm )
      verifier_index]

  let wrap_verification_key_to_yojson =
    [%to_yojson:
      ( Backend.Tock.Field.t
      , srs
      , Backend.Tick.Field.t or_infinity poly_comm )
      verifier_index]
end

let empty () = ref Yojson_map.empty

let get_proof t ~verification_key ~public_input =
  let open Option.Let_syntax in
  let%bind for_vk = Map.find !t verification_key in
  Map.find for_vk public_input

let get_step_proof t ~keypair ~public_input =
  let open Option.Let_syntax in
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fp.get public_input i )
    |> [%to_yojson: Backend.Tick.Field.t array]
  in
  let verification_key =
    Backend.Tick.Keypair.vk keypair |> Json.step_verification_key_to_yojson
  in
  let%bind proof_json = get_proof t ~verification_key ~public_input in
  Option.try_with (fun () ->
      Result.ok_or_failwith @@ Backend.Tick.Proof.of_yojson proof_json )

let get_wrap_proof t ~keypair ~public_input =
  let open Option.Let_syntax in
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fq.get public_input i )
    |> [%to_yojson: Backend.Tock.Field.t array]
  in
  let verification_key =
    Backend.Tock.Keypair.vk keypair |> Json.wrap_verification_key_to_yojson
  in
  let%bind proof_json = get_proof t ~verification_key ~public_input in
  Option.try_with (fun () ->
      Result.ok_or_failwith @@ Backend.Tock.Proof.of_yojson proof_json )

let set_proof t ~verification_key ~public_input proof =
  t :=
    Map.update !t verification_key ~f:(function
      | None ->
          Yojson_map.singleton public_input proof
      | Some for_vk ->
          Map.set for_vk ~key:public_input ~data:proof )

let set_step_proof t ~keypair ~public_input proof =
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fp.get public_input i )
    |> [%to_yojson: Backend.Tick.Field.t array]
  in
  let verification_key =
    Backend.Tick.Keypair.vk keypair |> Json.step_verification_key_to_yojson
  in
  let proof_json = Backend.Tick.Proof.to_yojson proof in
  set_proof t ~verification_key ~public_input proof_json

let set_wrap_proof t ~keypair ~public_input proof =
  let public_input =
    let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
    Array.init len ~f:(fun i ->
        Kimchi_bindings.FieldVectors.Fq.get public_input i )
    |> [%to_yojson: Backend.Tock.Field.t array]
  in
  let verification_key =
    Backend.Tock.Keypair.vk keypair |> Json.wrap_verification_key_to_yojson
  in
  let proof_json = Backend.Tock.Proof.to_yojson proof in
  set_proof t ~verification_key ~public_input proof_json

let is_env_var_set_requesting_error_for_proofs () =
  match Sys.getenv_opt "ERROR_ON_PROOF" with
  | Some ("true" | "t" (* insert whatever value is okay here *)) ->
      true
  | None | Some _ ->
      false

1	open Core_kernel	22✔
2
3	[@@@warning "-4-27"]
4
5	module Yojson_map = Map.Make (struct
UNCOV 6	type t =	×
7	[ `Null	×
8	\| `Bool of bool	×
9	\| `Int of int	×
10	\| `Intlit of string	×
11	\| `Float of float	×
12	\| `String of string	×
13	\| `Assoc of (string * t) list	×
UNCOV 14	\| `List of t list	×
15	\| `Tuple of t list	×
16	\| `Variant of string * t option ]	×
17	[@@deriving compare, sexp]	×
18	end)
19
20	[@@@warning "+4+27"]
21
22	type t = Yojson.Safe.t Yojson_map.t Yojson_map.t ref
23
24	(* We use a slightly more verbose format here, so that it's easy to debug.
25	There are some overheads to handling this, but the amount of computation we
26	save by caching the proofs is orders of magnitude higher, so it's not really
27	an issue.
28	*)
29	let to_yojson t =
30	`List	×
31	(Map.fold ~init:[] !t ~f:(fun ~key ~data xs ->	×
32	let proofs =	×
33	Map.fold ~init:[] data ~f:(fun ~key ~data xs ->
34	`Assoc [ ("public_input", key); ("proof", data) ] :: xs )	×
35	in
36	`Assoc [ ("verification_key", key); ("proofs", `List proofs) ] :: xs )	×
37	)
38
39	(* This mirrors the format of [to_yojson], carefully ensuring that we can
40	decode what we encode, and reporting an error when the format differs from
41	what we expect.
42
43	Note that, since this is a cache, it should always be possible to regenerate
44	proofs for the cache by starting with the empty cache and calling
45	[to_yojson] on the result.
46	*)
47	let of_yojson t =
UNCOV 48	Result.try_with (fun () ->	×
UNCOV 49	match t with	×
UNCOV 50	\| `List xs ->	×
51	let for_vks =
52	List.map xs ~f:(function
UNCOV 53	\| `Assoc [ ("verification_key", key); ("proofs", `List proofs) ]	×
54	->
55	let proofs =
56	List.map proofs ~f:(function
UNCOV 57	\| `Assoc [ ("public_input", key); ("proof", data) ] ->	×
58	(key, data)
59	\| _ ->	×
60	failwith
61	"Expected fields `public_input`, `proof` as a \
62	record in that order; received something \
63	different" )
64	in
UNCOV 65	(key, Yojson_map.of_alist_exn proofs)	×
66	\| _ ->	×
67	failwith
68	"Expected fields `verification_key`, `proofs` as a record \
69	in that order, where `proofs` is a list; received \
70	something different" )
71	in
UNCOV 72	ref (Yojson_map.of_alist_exn for_vks)	×
73	\| _ ->	×
74	failwith "Expected a list, got something different" )
75	\|> Result.map_error ~f:Exn.to_string
76
77	(* Alias types with a [deriving to_yojson] annotation, so that we don't have to
78	spell out the serialization explicitly.
79	*)
80	module Json = struct
UNCOV 81	type 'f or_infinity = 'f Kimchi_types.or_infinity =	×
82	\| Infinity
UNCOV 83	\| Finite of ('f * 'f)	×
84	[@@deriving to_yojson]
85
UNCOV 86	type 'caml_g poly_comm = 'caml_g Kimchi_types.poly_comm =	×
UNCOV 87	{ unshifted : 'caml_g array; shifted : 'caml_g option }	×
88	[@@deriving to_yojson]
89
90	type lookup_patterns = Kimchi_types.lookup_patterns =	×
91	{ xor : bool; lookup : bool; range_check : bool; foreign_field_mul : bool }	×
92	[@@deriving to_yojson]
93
94	type lookup_features = Kimchi_types.lookup_features =	×
95	{ patterns : lookup_patterns	×
96	; joint_lookup_used : bool	×
97	; uses_runtime_tables : bool	×
98	}
99	[@@deriving to_yojson]
100
101	type lookups_used = Kimchi_types.VerifierIndex.Lookup.lookups_used =	×
102	\| Single
103	\| Joint
104	[@@deriving to_yojson]
105
106	type lookup_info = Kimchi_types.VerifierIndex.Lookup.lookup_info =	×
107	{ max_per_row : int; max_joint_size : int; features : lookup_features }	×
108	[@@deriving to_yojson]
109
110	type 't lookup_selectors =	×
111	't Kimchi_types.VerifierIndex.Lookup.lookup_selectors =
112	{ lookup : 't option	×
113	; xor : 't option	×
114	; range_check : 't option	×
115	; ffmul : 't option	×
116	}
117	[@@deriving to_yojson]
118
119	type 'poly_comm lookup = 'poly_comm Kimchi_types.VerifierIndex.Lookup.t =	×
120	{ joint_lookup_used : bool	×
121	; lookup_table : 'poly_comm array	×
122	; lookup_selectors : 'poly_comm lookup_selectors	×
123	; table_ids : 'poly_comm option	×
124	; lookup_info : lookup_info	×
125	; runtime_tables_selector : 'poly_comm option	×
126	}
127	[@@deriving to_yojson]	×
128
UNCOV 129	type 'fr domain = 'fr Kimchi_types.VerifierIndex.domain =	×
UNCOV 130	{ log_size_of_group : int; group_gen : 'fr }	×
131	[@@deriving to_yojson]
132
UNCOV 133	type 'poly_comm verification_evals =	×
134	'poly_comm Kimchi_types.VerifierIndex.verification_evals =
UNCOV 135	{ sigma_comm : 'poly_comm array	×
UNCOV 136	; coefficients_comm : 'poly_comm array	×
UNCOV 137	; generic_comm : 'poly_comm	×
UNCOV 138	; psm_comm : 'poly_comm	×
UNCOV 139	; complete_add_comm : 'poly_comm	×
UNCOV 140	; mul_comm : 'poly_comm	×
UNCOV 141	; emul_comm : 'poly_comm	×
UNCOV 142	; endomul_scalar_comm : 'poly_comm	×
143	; xor_comm : 'poly_comm option [@default None]	×
144	; range_check0_comm : 'poly_comm option [@default None]	×
145	; range_check1_comm : 'poly_comm option [@default None]	×
146	; foreign_field_add_comm : 'poly_comm option [@default None]	×
147	; foreign_field_mul_comm : 'poly_comm option [@default None]	×
148	; rot_comm : 'poly_comm option [@default None]	×
149	}
150	[@@deriving to_yojson]
151
UNCOV 152	type ('fr, 'srs, 'poly_comm) verifier_index =	×
153	('fr, 'srs, 'poly_comm) Kimchi_types.VerifierIndex.verifier_index =
UNCOV 154	{ domain : 'fr domain	×
UNCOV 155	; max_poly_size : int	×
UNCOV 156	; public : int	×
UNCOV 157	; prev_challenges : int	×
UNCOV 158	; srs : 'srs	×
UNCOV 159	; evals : 'poly_comm verification_evals	×
UNCOV 160	; shifts : 'fr array	×
UNCOV 161	; lookup_index : 'poly_comm lookup option	×
162	; zk_rows : int [@default 3]	×
163	}
UNCOV 164	[@@deriving to_yojson]	×
165
UNCOV 166	let srs_to_yojson _ = `Null	×
167
168	let step_verification_key_to_yojson =
169	[%to_yojson:
UNCOV 170	( Backend.Tick.Field.t	×
UNCOV 171	, srs	×
UNCOV 172	, Backend.Tock.Field.t or_infinity poly_comm )	×
UNCOV 173	verifier_index]	×
174
175	let wrap_verification_key_to_yojson =
176	[%to_yojson:
UNCOV 177	( Backend.Tock.Field.t	×
UNCOV 178	, srs	×
UNCOV 179	, Backend.Tick.Field.t or_infinity poly_comm )	×
UNCOV 180	verifier_index]	×
181	end
182
183	let empty () = ref Yojson_map.empty	×
184
185	let get_proof t ~verification_key ~public_input =
UNCOV 186	let open Option.Let_syntax in	×
UNCOV 187	let%bind for_vk = Map.find !t verification_key in	×
UNCOV 188	Map.find for_vk public_input	×
189
190	let get_step_proof t ~keypair ~public_input =
UNCOV 191	let open Option.Let_syntax in	×
192	let public_input =
193	let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
UNCOV 194	Array.init len ~f:(fun i ->	×
UNCOV 195	Kimchi_bindings.FieldVectors.Fp.get public_input i )	×
UNCOV 196	\|> [%to_yojson: Backend.Tick.Field.t array]	×
197	in
198	let verification_key =
UNCOV 199	Backend.Tick.Keypair.vk keypair \|> Json.step_verification_key_to_yojson	×
200	in
UNCOV 201	let%bind proof_json = get_proof t ~verification_key ~public_input in	×
UNCOV 202	Option.try_with (fun () ->	×
UNCOV 203	Result.ok_or_failwith @@ Backend.Tick.Proof.of_yojson proof_json )	×
204
205	let get_wrap_proof t ~keypair ~public_input =
UNCOV 206	let open Option.Let_syntax in	×
207	let public_input =
208	let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
UNCOV 209	Array.init len ~f:(fun i ->	×
UNCOV 210	Kimchi_bindings.FieldVectors.Fq.get public_input i )	×
UNCOV 211	\|> [%to_yojson: Backend.Tock.Field.t array]	×
212	in
213	let verification_key =
UNCOV 214	Backend.Tock.Keypair.vk keypair \|> Json.wrap_verification_key_to_yojson	×
215	in
UNCOV 216	let%bind proof_json = get_proof t ~verification_key ~public_input in	×
UNCOV 217	Option.try_with (fun () ->	×
UNCOV 218	Result.ok_or_failwith @@ Backend.Tock.Proof.of_yojson proof_json )	×
219
220	let set_proof t ~verification_key ~public_input proof =
221	t :=	×
222	Map.update !t verification_key ~f:(function	×
223	\| None ->	×
224	Yojson_map.singleton public_input proof
225	\| Some for_vk ->	×
226	Map.set for_vk ~key:public_input ~data:proof )
227
228	let set_step_proof t ~keypair ~public_input proof =
229	let public_input =	×
230	let len = Kimchi_bindings.FieldVectors.Fp.length public_input in
231	Array.init len ~f:(fun i ->	×
232	Kimchi_bindings.FieldVectors.Fp.get public_input i )	×
233	\|> [%to_yojson: Backend.Tick.Field.t array]	×
234	in
235	let verification_key =
236	Backend.Tick.Keypair.vk keypair \|> Json.step_verification_key_to_yojson	×
237	in
238	let proof_json = Backend.Tick.Proof.to_yojson proof in	×
239	set_proof t ~verification_key ~public_input proof_json	×
240
241	let set_wrap_proof t ~keypair ~public_input proof =
242	let public_input =	×
243	let len = Kimchi_bindings.FieldVectors.Fq.length public_input in
244	Array.init len ~f:(fun i ->	×
245	Kimchi_bindings.FieldVectors.Fq.get public_input i )	×
246	\|> [%to_yojson: Backend.Tock.Field.t array]	×
247	in
248	let verification_key =
249	Backend.Tock.Keypair.vk keypair \|> Json.wrap_verification_key_to_yojson	×
250	in
251	let proof_json = Backend.Tock.Proof.to_yojson proof in	×
252	set_proof t ~verification_key ~public_input proof_json	×
253
254	let is_env_var_set_requesting_error_for_proofs () =
255	match Sys.getenv_opt "ERROR_ON_PROOF" with	×
256	\| Some ("true" \| "t" (* insert whatever value is okay here *)) ->	×
257	true
258	\| None \| Some _ ->	×
259	false	44✔

MinaProtocol / mina / 2903

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous