3438

Committed 06 Mar 2025 07:52PM UTC coverage: 32.762% (-28.0%) from 60.748%

Build # 3438

Build Type

Pull #16698

buildkite

Committed by

dkijania

Commit Message

Revert "unify gsutil upload scripts"

This reverts commit aac20d7c4.

Pull Request Pull Request #16698: [CI] Unify buildkite cache access

Run Details

23422 of 71492 relevant lines covered (32.76%)

18618.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

6.35

/src/lib/disk_cache/test_lib/disk_cache_test_lib.ml

open Async
open Core

module Mock = struct
  type t = { proof : Bounded_types.String.Stable.V1.t }
  [@@deriving bin_io_unversioned]
end

module type S = sig
  module Cache : sig
    type t

    type id
  end

  val logger : Logger.t

  val simple_write :
       ?additional_checks:
         (Cache.t * Mock.t * Mock.t * Cache.id * Cache.id -> unit)
    -> unit
    -> unit

  val initialization_special_cases : unit -> unit

  val remove_data_on_gc : unit -> unit
end

module type S_extended = sig
  include S

  val simple_write_with_iteration : unit -> unit
end

module Make_impl (Cache : Disk_cache_intf.S with module Data := Mock) :
  S with module Cache := Cache = struct
  let () =
    Core.Backtrace.elide := false ;
    Async.Scheduler.set_record_backtraces true

  let logger = Logger.null ()

  let initialize_cache_or_fail tmpd ~logger =
    let open Deferred.Let_syntax in
    let%bind cache_res = Cache.initialize tmpd ~logger in
    match cache_res with
    | Ok cache ->
        return cache
    | Error _err ->
        failwith "error during initialization"

  let simple_write_impl ?(additional_checks = const ()) tmp_dir =
    let%map cache = initialize_cache_or_fail tmp_dir ~logger in

    let proof1 = Mock.{ proof = "dummy" } in
    let proof2 = Mock.{ proof = "smart" } in

    let id1 = Cache.put cache proof1 in
    let id2 = Cache.put cache proof2 in

    additional_checks (cache, proof1, proof2, id1, id2) ;

    [%test_eq: int] (Cache.count cache) 2
      ~message:"cache should contain only 2 elements" ;

    let proof_from_cache1 = Cache.get cache id1 in
    [%test_eq: string] proof1.proof proof_from_cache1.proof
      ~message:"invalid proof from cache" ;

    let proof_from_cache2 = Cache.get cache id2 in
    [%test_eq: string] proof2.proof proof_from_cache2.proof
      ~message:"invalid proof from cache"

  let simple_write ?additional_checks () =
    Async.Thread_safe.block_on_async_exn
    @@ fun () ->
    File_system.with_temp_dir "disk_cache"
      ~f:(simple_write_impl ?additional_checks)

  let remove_data_on_gc_impl tmp_dir =
    let%map cache = initialize_cache_or_fail tmp_dir ~logger in

    let proof = Mock.{ proof = "dummy" } in

    (let id = Cache.put cache proof in

     [%test_eq: int] (Cache.count cache) 1
       ~message:"cache should contain only 1 element" ;

     let proof_from_cache = Cache.get cache id in
     [%test_eq: string] proof.proof proof_from_cache.proof
       ~message:"invalid proof from cache" ) ;

    Gc.compact () ;

    [%test_eq: int] (Cache.count cache) 0
      ~message:"cache should be empty after garbage collector run"

  let remove_data_on_gc () =
    Async.Thread_safe.block_on_async_exn
    @@ fun () ->
    File_system.with_temp_dir "disk_cache-remove_data_on_gc"
      ~f:remove_data_on_gc_impl

  let initialize_and_expect_failure path ~logger =
    let%bind cache_res = Cache.initialize path ~logger in
    match cache_res with
    | Ok _ ->
        failwith "unexpected initialization success"
    | Error _err ->
        return ()

  let initialization_special_cases_impl tmp_dir =
    (* create a directory with 0x000 permissions and initialize from it *)
    let%bind () =
      let perm_denied_dir = tmp_dir ^/ "permission_denied" in
      Core.Unix.mkdir ~perm:0o000 perm_denied_dir ;
      let unreachable = perm_denied_dir ^/ "some_unreachable_path" in
      initialize_and_expect_failure unreachable ~logger
    in

    (* create a directory, create a symlink to it and initialize from a synlimk *)
    let%bind () =
      let some_dir_name = "some_dir" in
      let some_dir = tmp_dir ^/ some_dir_name in
      Core.Unix.mkdir some_dir ;
      let dir_symlink = tmp_dir ^/ "dir_link" in
      Core.Unix.symlink ~target:some_dir_name ~link_name:dir_symlink ;
      Cache.initialize dir_symlink ~logger
      >>| function
      | Ok _ ->
          ()
      | Error _ ->
          failwith "unexpected initialization failure for dir symlink"
    in

    (* create a symlink to a non-existent file, try to initialize from symlink *)
    let%bind () =
      let corrupt_symlink = tmp_dir ^/ "corrupt_link" in
      Core.Unix.symlink ~target:"doesnt_exist" ~link_name:corrupt_symlink ;
      initialize_and_expect_failure corrupt_symlink ~logger
    in

    (* create a file and initialize from it *)
    let%bind some_file_name =
      let some_file_name = "file.txt" in
      let some_file = tmp_dir ^/ some_file_name in
      Out_channel.write_all some_file ~data:"yo" ;
      initialize_and_expect_failure some_file ~logger >>| const some_file_name
    in

    (* create a symlink to an existing file, try to initialize from symlink *)
    let symlink = tmp_dir ^/ "link" in
    Core.Unix.symlink ~target:some_file_name ~link_name:symlink ;
    initialize_and_expect_failure symlink ~logger

  let initialization_special_cases () =
    Async.Thread_safe.block_on_async_exn
    @@ fun () ->
    File_system.with_temp_dir "disk_cache-invalid-initialization"
      ~f:initialization_special_cases_impl
end

module Make (Disk_cache : Disk_cache_intf.F) :
  S with module Cache := Disk_cache(Mock) = struct
  include Make_impl (Disk_cache (Mock))
end

module Make_extended (Disk_cache : Disk_cache_intf.F_extended) :
  S_extended with module Cache := Disk_cache(Mock) = struct
  module Cache = Disk_cache (Mock)
  include Make_impl (Cache)

  let iteration_checks (cache, proof1, proof2, id1, id2) =
    let id1_not_visited = ref true in
    let id2_not_visited = ref true in
    Cache.iteri cache ~f:(fun id content ->
        let expected_content =
          if id = Cache.int_of_id id1 then (
            assert !id1_not_visited ;
            id1_not_visited := false ;
            proof1 )
          else if id = Cache.int_of_id id2 then (
            assert !id2_not_visited ;
            id2_not_visited := false ;
            proof2 )
          else failwith "unexpected key in iteration"
        in
        [%test_eq: string] content.Mock.proof expected_content.Mock.proof ;
        `Continue ) ;
    assert ((not !id1_not_visited) && not !id2_not_visited)

  let simple_write_with_iteration =
    simple_write ~additional_checks:iteration_checks
end

1	open Async	18✔
2	open Core
3
4	module Mock = struct
5	type t = { proof : Bounded_types.String.Stable.V1.t }	18✔
6	[@@deriving bin_io_unversioned]	72✔
7	end
8
9	module type S = sig
10	module Cache : sig
11	type t
12
13	type id
14	end
15
16	val logger : Logger.t
17
18	val simple_write :
19	?additional_checks:
20	(Cache.t * Mock.t * Mock.t * Cache.id * Cache.id -> unit)
21	-> unit
22	-> unit
23
24	val initialization_special_cases : unit -> unit
25
26	val remove_data_on_gc : unit -> unit
27	end
28
29	module type S_extended = sig
30	include S
31
32	val simple_write_with_iteration : unit -> unit
33	end
34
35	module Make_impl (Cache : Disk_cache_intf.S with module Data := Mock) :
36	S with module Cache := Cache = struct
37	let () =
38	Core.Backtrace.elide := false ;
39	Async.Scheduler.set_record_backtraces true	×
40
41	let logger = Logger.null ()	×
42
43	let initialize_cache_or_fail tmpd ~logger =
44	let open Deferred.Let_syntax in	×
45	let%bind cache_res = Cache.initialize tmpd ~logger in	×
46	match cache_res with	×
47	\| Ok cache ->	×
48	return cache
49	\| Error _err ->	×
50	failwith "error during initialization"
51
52	let simple_write_impl ?(additional_checks = const ()) tmp_dir =	×
53	let%map cache = initialize_cache_or_fail tmp_dir ~logger in	×
54
55	let proof1 = Mock.{ proof = "dummy" } in	×
56	let proof2 = Mock.{ proof = "smart" } in
57
58	let id1 = Cache.put cache proof1 in
59	let id2 = Cache.put cache proof2 in	×
60
61	additional_checks (cache, proof1, proof2, id1, id2) ;	×
62
63	[%test_eq: int] (Cache.count cache) 2	×
64	~message:"cache should contain only 2 elements" ;
65
66	let proof_from_cache1 = Cache.get cache id1 in	×
67	[%test_eq: string] proof1.proof proof_from_cache1.proof	×
68	~message:"invalid proof from cache" ;
69
70	let proof_from_cache2 = Cache.get cache id2 in	×
71	[%test_eq: string] proof2.proof proof_from_cache2.proof	×
72	~message:"invalid proof from cache"
73
74	let simple_write ?additional_checks () =
75	Async.Thread_safe.block_on_async_exn	×
76	@@ fun () ->
77	File_system.with_temp_dir "disk_cache"	×
78	~f:(simple_write_impl ?additional_checks)
79
80	let remove_data_on_gc_impl tmp_dir =
81	let%map cache = initialize_cache_or_fail tmp_dir ~logger in	×
82
83	let proof = Mock.{ proof = "dummy" } in	×
84
85	(let id = Cache.put cache proof in
86
87	[%test_eq: int] (Cache.count cache) 1	×
88	~message:"cache should contain only 1 element" ;
89
90	let proof_from_cache = Cache.get cache id in	×
91	[%test_eq: string] proof.proof proof_from_cache.proof	×
92	~message:"invalid proof from cache" ) ;
93
94	Gc.compact () ;
95
96	[%test_eq: int] (Cache.count cache) 0	×
97	~message:"cache should be empty after garbage collector run"
98
99	let remove_data_on_gc () =
100	Async.Thread_safe.block_on_async_exn	×
101	@@ fun () ->
102	File_system.with_temp_dir "disk_cache-remove_data_on_gc"	×
103	~f:remove_data_on_gc_impl
104
105	let initialize_and_expect_failure path ~logger =
106	let%bind cache_res = Cache.initialize path ~logger in	×
107	match cache_res with	×
108	\| Ok _ ->	×
109	failwith "unexpected initialization success"
110	\| Error _err ->	×
111	return ()
112
113	let initialization_special_cases_impl tmp_dir =
114	(* create a directory with 0x000 permissions and initialize from it *)
115	let%bind () =
116	let perm_denied_dir = tmp_dir ^/ "permission_denied" in
117	Core.Unix.mkdir ~perm:0o000 perm_denied_dir ;	×
118	let unreachable = perm_denied_dir ^/ "some_unreachable_path" in	×
119	initialize_and_expect_failure unreachable ~logger	×
120	in
121
122	(* create a directory, create a symlink to it and initialize from a synlimk *)
123	let%bind () =
124	let some_dir_name = "some_dir" in
125	let some_dir = tmp_dir ^/ some_dir_name in
126	Core.Unix.mkdir some_dir ;	×
127	let dir_symlink = tmp_dir ^/ "dir_link" in	×
128	Core.Unix.symlink ~target:some_dir_name ~link_name:dir_symlink ;	×
129	Cache.initialize dir_symlink ~logger	×
130	>>\| function	×
131	\| Ok _ ->	×
132	()
133	\| Error _ ->	×
134	failwith "unexpected initialization failure for dir symlink"
135	in
136
137	(* create a symlink to a non-existent file, try to initialize from symlink *)
138	let%bind () =
139	let corrupt_symlink = tmp_dir ^/ "corrupt_link" in
140	Core.Unix.symlink ~target:"doesnt_exist" ~link_name:corrupt_symlink ;	×
141	initialize_and_expect_failure corrupt_symlink ~logger	×
142	in
143
144	(* create a file and initialize from it *)
145	let%bind some_file_name =
146	let some_file_name = "file.txt" in
147	let some_file = tmp_dir ^/ some_file_name in
148	Out_channel.write_all some_file ~data:"yo" ;	×
149	initialize_and_expect_failure some_file ~logger >>\| const some_file_name	×
150	in
151
152	(* create a symlink to an existing file, try to initialize from symlink *)
153	let symlink = tmp_dir ^/ "link" in	×
154	Core.Unix.symlink ~target:some_file_name ~link_name:symlink ;	×
155	initialize_and_expect_failure symlink ~logger
156
157	let initialization_special_cases () =
158	Async.Thread_safe.block_on_async_exn	×
159	@@ fun () ->
160	File_system.with_temp_dir "disk_cache-invalid-initialization"	×
161	~f:initialization_special_cases_impl
162	end
163
164	module Make (Disk_cache : Disk_cache_intf.F) :
165	S with module Cache := Disk_cache(Mock) = struct
166	include Make_impl (Disk_cache (Mock))
167	end
168
169	module Make_extended (Disk_cache : Disk_cache_intf.F_extended) :
170	S_extended with module Cache := Disk_cache(Mock) = struct
171	module Cache = Disk_cache (Mock)
172	include Make_impl (Cache)
173
174	let iteration_checks (cache, proof1, proof2, id1, id2) =
175	let id1_not_visited = ref true in	×
176	let id2_not_visited = ref true in
177	Cache.iteri cache ~f:(fun id content ->
178	let expected_content =	×
179	if id = Cache.int_of_id id1 then (	×
180	assert !id1_not_visited ;	×
181	id1_not_visited := false ;
182	proof1 )
183	else if id = Cache.int_of_id id2 then (	×
184	assert !id2_not_visited ;	×
185	id2_not_visited := false ;
186	proof2 )
187	else failwith "unexpected key in iteration"	×
188	in
189	[%test_eq: string] content.Mock.proof expected_content.Mock.proof ;	×
190	`Continue ) ;	×
191	assert ((not !id1_not_visited) && not !id2_not_visited)	×
192
193	let simple_write_with_iteration =
194	simple_write ~additional_checks:iteration_checks
195	end	36✔

MinaProtocol / mina / 3438

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous