2903

Committed 15 Nov 2024 01:59PM UTC coverage: 36.723% (-25.0%) from 61.682%

Build # 2903

Build Type

Pull #16342

buildkite

Committed by

dkijania

Commit Message

Merge branch 'dkijania/remove_publish_job_from_pr_comp' into dkijania/remove_publish_job_from_pr_dev

Pull Request Pull Request #16342: [DEV] Publish debians only on nightly and stable

Run Details

15 of 40 new or added lines in 14 files covered. (37.5%)

15175 existing lines in 340 files now uncovered.

24554 of 66863 relevant lines covered (36.72%)

20704.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.83

/src/lib/rocksdb/database.ml

(* rocksdb.ml -- expose RocksDB operations for Mina *)

type t = { uuid : Uuid.Stable.V1.t; db : (Rocks.t[@sexp.opaque]) }
[@@deriving sexp]

let create directory =
  let opts = Rocks.Options.create () in
  Rocks.Options.set_create_if_missing opts true ;
  Rocks.Options.set_prefix_extractor opts
    (Rocks.Options.SliceTransform.Noop.create_no_gc ()) ;
  { uuid = Uuid_unix.create (); db = Rocks.open_db ~opts directory }

let create_checkpoint t dir =
  Rocks.checkpoint_create t.db ~dir ?log_size_for_flush:None () ;
  create dir

let make_checkpoint t dir =
  Rocks.checkpoint_create t.db ~dir ?log_size_for_flush:None ()

let get_uuid t = t.uuid

let close t = Rocks.close t.db

let get t ~(key : Bigstring.t) : Bigstring.t option =
  Rocks.get ?pos:None ?len:None ?opts:None t.db key

let get_batch t ~(keys : Bigstring.t list) : Bigstring.t option list =
  Rocks.multi_get t.db keys

let set t ~(key : Bigstring.t) ~(data : Bigstring.t) : unit =
  Rocks.put ?key_pos:None ?key_len:None ?value_pos:None ?value_len:None
    ?opts:None t.db key data

let[@warning "-16"] set_batch t ?(remove_keys : Bigstring.t list = [])
    ~(key_data_pairs : (Bigstring.t * Bigstring.t) list) : unit =
  let batch = Rocks.WriteBatch.create () in
  (* write to batch *)
  List.iter key_data_pairs ~f:(fun (key, data) ->
      Rocks.WriteBatch.put batch key data ) ;
  (* Delete any key pairs *)
  List.iter remove_keys ~f:(fun key -> Rocks.WriteBatch.delete batch key) ;
  (* commit batch *)
  Rocks.write t.db batch

module Batch = struct
  type t = Rocks.WriteBatch.t

  let remove t ~key = Rocks.WriteBatch.delete t key

  let set t ~key ~data = Rocks.WriteBatch.put t key data

  let with_batch t ~f =
    let batch = Rocks.WriteBatch.create () in
    let result = f batch in
    Rocks.write t.db batch ; result
end

let remove t ~(key : Bigstring.t) : unit =
  Rocks.delete ?pos:None ?len:None ?opts:None t.db key

let copy_bigstring t : Bigstring.t =
  let tlen = Bigstring.length t in
  let new_t = Bigstring.create tlen in
  Bigstring.blit ~src:t ~dst:new_t ~src_pos:0 ~dst_pos:0 ~len:tlen ;
  new_t

let to_alist t : (Bigstring.t * Bigstring.t) list =
  let iterator = Rocks.Iterator.create t.db in
  Rocks.Iterator.seek_to_last iterator ;
  (* iterate backwards and cons, to build list sorted by key *)
  let rec loop accum =
    if Rocks.Iterator.is_valid iterator then (
      let key = copy_bigstring (Rocks.Iterator.get_key iterator) in
      let value = copy_bigstring (Rocks.Iterator.get_value iterator) in
      Rocks.Iterator.prev iterator ;
      loop ((key, value) :: accum) )
    else accum
  in
  loop []

let foldi :
       t
    -> init:'a
    -> f:(int -> 'a -> key:Bigstring.t -> data:Bigstring.t -> 'a)
    -> 'a =
 fun t ~init ~f ->
  let iterator = Rocks.Iterator.create t.db in
  Rocks.Iterator.seek_to_first iterator ;
  let rec loop i accum =
    if Rocks.Iterator.is_valid iterator then (
      let key = copy_bigstring (Rocks.Iterator.get_key iterator) in
      let data = copy_bigstring (Rocks.Iterator.get_value iterator) in
      Rocks.Iterator.next iterator ;
      loop (i + 1) (f i accum ~key ~data) )
    else accum
  in
  loop 0 init

let fold_until :
       t
    -> init:'a
    -> f:
         (   'a
          -> key:Bigstring.t
          -> data:Bigstring.t
          -> ('a, 'b) Continue_or_stop.t )
    -> finish:('a -> 'b)
    -> 'b =
 fun t ~init ~f ~finish ->
  let iterator = Rocks.Iterator.create t.db in
  Rocks.Iterator.seek_to_first iterator ;
  let rec loop accum =
    if Rocks.Iterator.is_valid iterator then (
      let key = copy_bigstring (Rocks.Iterator.get_key iterator) in
      let data = copy_bigstring (Rocks.Iterator.get_value iterator) in
      Rocks.Iterator.next iterator ;
      match f accum ~key ~data with Stop _ -> accum | Continue v -> loop v )
    else accum
  in
  finish @@ loop init

let to_bigstring = Bigstring.of_string

let%test_unit "get_batch" =
  Async.Thread_safe.block_on_async_exn (fun () ->
      File_system.with_temp_dir "/tmp/mina-rocksdb-test" ~f:(fun db_dir ->
          let db = create db_dir in
          let[@warning "-8"] [ key1; key2; key3 ] =
            List.map ~f:(fun s -> Bigstring.of_string s) [ "a"; "b"; "c" ]
          in
          let data = Bigstring.of_string "test" in
          set db ~key:key1 ~data ;
          set db ~key:key3 ~data ;
          let[@warning "-8"] [ res1; res2; res3 ] =
            get_batch db ~keys:[ key1; key2; key3 ]
          in
          assert ([%equal: Bigstring.t option] res1 (Some data)) ;
          assert ([%equal: Bigstring.t option] res2 None) ;
          assert ([%equal: Bigstring.t option] res3 (Some data)) ;
          Async.Deferred.unit ) )

let%test_unit "to_alist (of_alist l) = l" =
  Async.Thread_safe.block_on_async_exn
  @@ fun () ->
  Async.Quickcheck.async_test ~trials:20
    Quickcheck.Generator.(
      tuple2 String.quickcheck_generator String.quickcheck_generator |> list)
    ~f:(fun kvs ->
      match Hashtbl.of_alist (module String) kvs with
      | `Duplicate_key _ ->
          Async.Deferred.unit
      | `Ok _ ->
          File_system.with_temp_dir "/tmp/mina-rocksdb-test" ~f:(fun db_dir ->
              let sorted =
                List.sort kvs ~compare:[%compare: string * string]
                |> List.map ~f:(fun (k, v) -> (to_bigstring k, to_bigstring v))
              in
              let db = create db_dir in
              List.iter sorted ~f:(fun (key, data) -> set db ~key ~data) ;
              let alist =
                List.sort (to_alist db)
                  ~compare:[%compare: Bigstring.t * Bigstring.t]
              in
              [%test_result: (Bigstring.t * Bigstring.t) list] ~expect:sorted
                alist ;
              close db ;
              Async.Deferred.unit ) )

let%test_unit "checkpoint read" =
  let open Async in
  Thread_safe.block_on_async_exn
  @@ fun () ->
  Quickcheck.async_test ~trials:20
    Quickcheck.Generator.(
      list @@ tuple2 String.quickcheck_generator String.quickcheck_generator)
    ~f:(fun kvs ->
      match Hashtbl.of_alist (module String) kvs with
      | `Duplicate_key _ ->
          Deferred.unit
      | `Ok db_hashtbl -> (
          let open Core in
          let cp_hashtbl = Hashtbl.copy db_hashtbl in
          let db_dir = Filename.temp_dir "test_db" "" in
          let cp_dir =
            Filename.temp_dir_name ^/ "test_cp"
            ^ String.init 16 ~f:(fun _ -> (Int.to_string (Random.int 10)).[0])
          in
          let db = create db_dir in
          Hashtbl.iteri db_hashtbl ~f:(fun ~key ~data ->
              set db ~key:(to_bigstring key) ~data:(to_bigstring data) ) ;
          let cp = create_checkpoint db cp_dir in
          match
            ( Hashtbl.add db_hashtbl ~key:"db_key" ~data:"db_data"
            , Hashtbl.add cp_hashtbl ~key:"cp_key" ~data:"cp_data" )
          with
          | `Ok, `Ok ->
              set db ~key:(to_bigstring "db_key") ~data:(to_bigstring "db_data") ;
              set cp ~key:(to_bigstring "cp_key") ~data:(to_bigstring "cp_data") ;
              let db_sorted =
                List.sort
                  (Hashtbl.to_alist db_hashtbl)
                  ~compare:[%compare: string * string]
                |> List.map ~f:(fun (k, v) -> (to_bigstring k, to_bigstring v))
              in
              let cp_sorted =
                List.sort
                  (Hashtbl.to_alist cp_hashtbl)
                  ~compare:[%compare: string * string]
                |> List.map ~f:(fun (k, v) -> (to_bigstring k, to_bigstring v))
              in
              let db_alist =
                List.sort (to_alist db)
                  ~compare:[%compare: Bigstring.t * Bigstring.t]
              in
              let cp_alist =
                List.sort (to_alist cp)
                  ~compare:[%compare: Bigstring.t * Bigstring.t]
              in
              [%test_result: (Bigstring.t * Bigstring.t) list] ~expect:db_sorted
                db_alist ;
              [%test_result: (Bigstring.t * Bigstring.t) list] ~expect:cp_sorted
                cp_alist ;
              close db ;
              close cp ;
              Deferred.unit
          | _ ->
              Deferred.unit ) )

1	(* rocksdb.ml -- expose RocksDB operations for Mina *)
2		19✔
3	type t = { uuid : Uuid.Stable.V1.t; db : (Rocks.t[@sexp.opaque]) }	×
4	[@@deriving sexp]
5
6	let create directory =
7	let opts = Rocks.Options.create () in	73✔
8	Rocks.Options.set_create_if_missing opts true ;	73✔
9	Rocks.Options.set_prefix_extractor opts	73✔
10	(Rocks.Options.SliceTransform.Noop.create_no_gc ()) ;	73✔
11	{ uuid = Uuid_unix.create (); db = Rocks.open_db ~opts directory }	73✔
12
13	let create_checkpoint t dir =
14	Rocks.checkpoint_create t.db ~dir ?log_size_for_flush:None () ;	×
15	create dir	×
16
17	let make_checkpoint t dir =
18	Rocks.checkpoint_create t.db ~dir ?log_size_for_flush:None ()	×
19
20	let get_uuid t = t.uuid	×
21
22	let close t = Rocks.close t.db	18✔
23
24	let get t ~(key : Bigstring.t) : Bigstring.t option =
25	Rocks.get ?pos:None ?len:None ?opts:None t.db key	18,877✔
26
27	let get_batch t ~(keys : Bigstring.t list) : Bigstring.t option list =
28	Rocks.multi_get t.db keys	3,602✔
29
30	let set t ~(key : Bigstring.t) ~(data : Bigstring.t) : unit =
31	Rocks.put ?key_pos:None ?key_len:None ?value_pos:None ?value_len:None	437✔
32	?opts:None t.db key data
33
34	let[@warning "-16"] set_batch t ?(remove_keys : Bigstring.t list = [])	×
35	~(key_data_pairs : (Bigstring.t * Bigstring.t) list) : unit =
36	let batch = Rocks.WriteBatch.create () in	136✔
37	(* write to batch *)
38	List.iter key_data_pairs ~f:(fun (key, data) ->	136✔
39	Rocks.WriteBatch.put batch key data ) ;	5,234✔
40	(* Delete any key pairs *)
41	List.iter remove_keys ~f:(fun key -> Rocks.WriteBatch.delete batch key) ;	×
42	(* commit batch *)
43	Rocks.write t.db batch	136✔
44
45	module Batch = struct
46	type t = Rocks.WriteBatch.t
47
UNCOV 48	let remove t ~key = Rocks.WriteBatch.delete t key	×
49
50	let set t ~key ~data = Rocks.WriteBatch.put t key data	36✔
51
52	let with_batch t ~f =
53	let batch = Rocks.WriteBatch.create () in	6✔
54	let result = f batch in	6✔
55	Rocks.write t.db batch ; result	6✔
56	end
57
58	let remove t ~(key : Bigstring.t) : unit =
59	Rocks.delete ?pos:None ?len:None ?opts:None t.db key	×
60
61	let copy_bigstring t : Bigstring.t =
62	let tlen = Bigstring.length t in	×
63	let new_t = Bigstring.create tlen in	×
64	Bigstring.blit ~src:t ~dst:new_t ~src_pos:0 ~dst_pos:0 ~len:tlen ;	×
65	new_t
66
67	let to_alist t : (Bigstring.t * Bigstring.t) list =
68	let iterator = Rocks.Iterator.create t.db in	18✔
69	Rocks.Iterator.seek_to_last iterator ;	18✔
70	(* iterate backwards and cons, to build list sorted by key *)
71	let rec loop accum =	18✔
72	if Rocks.Iterator.is_valid iterator then (	×
73	let key = copy_bigstring (Rocks.Iterator.get_key iterator) in	×
74	let value = copy_bigstring (Rocks.Iterator.get_value iterator) in	×
75	Rocks.Iterator.prev iterator ;	×
76	loop ((key, value) :: accum) )	×
77	else accum	18✔
78	in
79	loop []
80
81	let foldi :
82	t
83	-> init:'a
84	-> f:(int -> 'a -> key:Bigstring.t -> data:Bigstring.t -> 'a)
85	-> 'a =
86	fun t ~init ~f ->
87	let iterator = Rocks.Iterator.create t.db in	×
88	Rocks.Iterator.seek_to_first iterator ;	×
89	let rec loop i accum =	×
90	if Rocks.Iterator.is_valid iterator then (	×
91	let key = copy_bigstring (Rocks.Iterator.get_key iterator) in	×
92	let data = copy_bigstring (Rocks.Iterator.get_value iterator) in	×
93	Rocks.Iterator.next iterator ;	×
94	loop (i + 1) (f i accum ~key ~data) )	×
95	else accum	×
96	in
97	loop 0 init
98
99	let fold_until :
100	t
101	-> init:'a
102	-> f:
103	( 'a
104	-> key:Bigstring.t
105	-> data:Bigstring.t
106	-> ('a, 'b) Continue_or_stop.t )
107	-> finish:('a -> 'b)
108	-> 'b =
109	fun t ~init ~f ~finish ->
110	let iterator = Rocks.Iterator.create t.db in	×
111	Rocks.Iterator.seek_to_first iterator ;	×
112	let rec loop accum =	×
113	if Rocks.Iterator.is_valid iterator then (	×
114	let key = copy_bigstring (Rocks.Iterator.get_key iterator) in	×
115	let data = copy_bigstring (Rocks.Iterator.get_value iterator) in	×
116	Rocks.Iterator.next iterator ;	×
117	match f accum ~key ~data with Stop _ -> accum \| Continue v -> loop v )	×
118	else accum	×
119	in
120	finish @@ loop init	×
121
122	let to_bigstring = Bigstring.of_string
123
124	let%test_unit "get_batch" =
125	Async.Thread_safe.block_on_async_exn (fun () ->	×
126	File_system.with_temp_dir "/tmp/mina-rocksdb-test" ~f:(fun db_dir ->	×
127	let db = create db_dir in	×
128	let[@warning "-8"] [ key1; key2; key3 ] =	×
129	List.map ~f:(fun s -> Bigstring.of_string s) [ "a"; "b"; "c" ]	×
130	in
131	let data = Bigstring.of_string "test" in	×
132	set db ~key:key1 ~data ;	×
133	set db ~key:key3 ~data ;	×
134	let[@warning "-8"] [ res1; res2; res3 ] =	×
135	get_batch db ~keys:[ key1; key2; key3 ]
136	in
137	assert ([%equal: Bigstring.t option] res1 (Some data)) ;	×
138	assert ([%equal: Bigstring.t option] res2 None) ;	×
139	assert ([%equal: Bigstring.t option] res3 (Some data)) ;	×
140	Async.Deferred.unit ) )
141
142	let%test_unit "to_alist (of_alist l) = l" =
143	Async.Thread_safe.block_on_async_exn	×
144	@@ fun () ->
145	Async.Quickcheck.async_test ~trials:20	×
146	Quickcheck.Generator.(
147	tuple2 String.quickcheck_generator String.quickcheck_generator \|> list)	×
148	~f:(fun kvs ->
149	match Hashtbl.of_alist (module String) kvs with	×
150	\| `Duplicate_key _ ->	×
151	Async.Deferred.unit
152	\| `Ok _ ->	×
153	File_system.with_temp_dir "/tmp/mina-rocksdb-test" ~f:(fun db_dir ->
154	let sorted =	×
155	List.sort kvs ~compare:[%compare: string * string]	×
156	\|> List.map ~f:(fun (k, v) -> (to_bigstring k, to_bigstring v))	×
157	in
158	let db = create db_dir in	×
159	List.iter sorted ~f:(fun (key, data) -> set db ~key ~data) ;	×
160	let alist =	×
161	List.sort (to_alist db)	×
162	~compare:[%compare: Bigstring.t * Bigstring.t]	×
163	in
164	[%test_result: (Bigstring.t * Bigstring.t) list] ~expect:sorted	×
165	alist ;
166	close db ;	×
167	Async.Deferred.unit ) )	×
168
169	let%test_unit "checkpoint read" =
170	let open Async in	×
171	Thread_safe.block_on_async_exn
172	@@ fun () ->
173	Quickcheck.async_test ~trials:20	×
174	Quickcheck.Generator.(
175	list @@ tuple2 String.quickcheck_generator String.quickcheck_generator)	×
176	~f:(fun kvs ->
177	match Hashtbl.of_alist (module String) kvs with	×
178	\| `Duplicate_key _ ->	×
179	Deferred.unit
180	\| `Ok db_hashtbl -> (	×
181	let open Core in
182	let cp_hashtbl = Hashtbl.copy db_hashtbl in
183	let db_dir = Filename.temp_dir "test_db" "" in	×
184	let cp_dir =	×
185	Filename.temp_dir_name ^/ "test_cp"
186	^ String.init 16 ~f:(fun _ -> (Int.to_string (Random.int 10)).[0])	×
187	in
188	let db = create db_dir in	×
189	Hashtbl.iteri db_hashtbl ~f:(fun ~key ~data ->	×
190	set db ~key:(to_bigstring key) ~data:(to_bigstring data) ) ;	×
191	let cp = create_checkpoint db cp_dir in	×
192	match	×
193	( Hashtbl.add db_hashtbl ~key:"db_key" ~data:"db_data"	×
194	, Hashtbl.add cp_hashtbl ~key:"cp_key" ~data:"cp_data" )	×
195	with
196	\| `Ok, `Ok ->	×
197	set db ~key:(to_bigstring "db_key") ~data:(to_bigstring "db_data") ;	×
198	set cp ~key:(to_bigstring "cp_key") ~data:(to_bigstring "cp_data") ;	×
199	let db_sorted =	×
200	List.sort	×
201	(Hashtbl.to_alist db_hashtbl)	×
202	~compare:[%compare: string * string]	×
203	\|> List.map ~f:(fun (k, v) -> (to_bigstring k, to_bigstring v))	×
204	in
205	let cp_sorted =	×
206	List.sort	×
207	(Hashtbl.to_alist cp_hashtbl)	×
208	~compare:[%compare: string * string]	×
209	\|> List.map ~f:(fun (k, v) -> (to_bigstring k, to_bigstring v))	×
210	in
211	let db_alist =	×
212	List.sort (to_alist db)	×
213	~compare:[%compare: Bigstring.t * Bigstring.t]	×
214	in
215	let cp_alist =	×
216	List.sort (to_alist cp)	×
217	~compare:[%compare: Bigstring.t * Bigstring.t]	×
218	in
219	[%test_result: (Bigstring.t * Bigstring.t) list] ~expect:db_sorted	×
220	db_alist ;
221	[%test_result: (Bigstring.t * Bigstring.t) list] ~expect:cp_sorted	×
222	cp_alist ;
223	close db ;	×
224	close cp ;	×
225	Deferred.unit	×
226	\| _ ->	×
227	Deferred.unit ) )	38✔

MinaProtocol / mina / 2903

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous