c7479091597c3382f92c87a4fd0395db3ee30193

Committed 30 May 2025 03:48PM UTC coverage: 57.42% (-0.3%) from 57.672%

Build # c7479091597c3382f92c87a4fd0395db3ee30193

Build Type

push

github

Committed by NSHkr

Commit Message

refactor expression_processors

Run Details

98 of 221 new or added lines in 6 files covered. (44.34%)

28 existing lines in 2 files now uncovered.

6210 of 10815 relevant lines covered (57.42%)

3137.51 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.75

/lib/elixir_scope/ast_repository/memory_manager/compressor.ex

defmodule ElixirScope.ASTRepository.MemoryManager.Compressor do
  @moduledoc """
  Data compression subsystem for compressing infrequently accessed analysis data.

  Uses binary term compression to reduce memory footprint of large AST
  structures and analysis results that are rarely accessed.
  """

  require Logger

  @type compression_stats :: %{
    modules_compressed: non_neg_integer(),
    compression_ratio: float(),
    space_saved_bytes: non_neg_integer(),
    last_compression_duration: non_neg_integer(),
    total_compressions: non_neg_integer()
  }

  @doc """
  Gets initial compression statistics structure.
  """
  @spec get_initial_stats() :: compression_stats()
  def get_initial_stats() do
    %{
      modules_compressed: 0,
      compression_ratio: 0.0,
      space_saved_bytes: 0,
      last_compression_duration: 0,
      total_compressions: 0
    }
  end

  @doc """
  Performs compression of infrequently accessed analysis data.

  ## Options

  - `:access_threshold` - Minimum access count to avoid compression (default: 5)
  - `:age_threshold` - Minimum age in seconds before compression (default: 1800)
  - `:compression_level` - Compression level 1-9 (default: 6)
  """
  @spec perform_compression(keyword()) :: {:ok, compression_stats()} | {:error, term()}
  def perform_compression(opts \\ []) do
    access_threshold = Keyword.get(opts, :access_threshold, 5)
    age_threshold = Keyword.get(opts, :age_threshold, 1800)
    compression_level = Keyword.get(opts, :compression_level, 6)

    # Validate compression level
    compression_level = case compression_level do
      level when is_integer(level) and level >= 1 and level <= 9 -> level
      _ -> 6  # Default to level 6 if invalid
    end

    current_time = System.monotonic_time(:second)
    cutoff_time = current_time - age_threshold

    try do
      # Find data to compress
      candidates = find_compression_candidates(cutoff_time, access_threshold)

      # Perform compression
      {compressed_count, total_original, total_compressed} = compress_candidates(candidates, compression_level)

      compression_ratio = if total_original > 0 do
        total_compressed / total_original
      else
        0.0
      end

      space_saved = total_original - total_compressed

      result = %{
        modules_compressed: compressed_count,
        compression_ratio: compression_ratio,
        space_saved_bytes: space_saved
      }

      Logger.info("Compression completed: #{compressed_count} modules, #{Float.round(compression_ratio * 100, 1)}% of original size")

      {:ok, result}
    rescue
      error ->
        Logger.error("Compression failed: #{inspect(error)}")
        {:error, {:compression_failed, error}}
    end
  end

  @doc """
  Updates compression statistics with new results.
  """
  @spec update_stats(compression_stats(), compression_stats(), non_neg_integer()) :: compression_stats()
  def update_stats(stats, result, duration) do
    %{stats |
      modules_compressed: stats.modules_compressed + result.modules_compressed,
      compression_ratio: result.compression_ratio,
      space_saved_bytes: stats.space_saved_bytes + result.space_saved_bytes,
      last_compression_duration: duration,
      total_compressions: stats.total_compressions + 1
    }
  end

  @doc """
  Compresses a single data structure using the specified compression level.
  """
  @spec compress_data(term(), integer()) :: {:ok, binary(), non_neg_integer(), non_neg_integer()} | {:error, term()}
  def compress_data(data, compression_level \\ 6) do
    try do
      # Convert to binary term format
      original_binary = :erlang.term_to_binary(data)
      original_size = byte_size(original_binary)

      # Compress using zlib
      compressed_binary = :zlib.compress(original_binary)
      compressed_size = byte_size(compressed_binary)

      {:ok, compressed_binary, original_size, compressed_size}
    rescue
      error ->
        {:error, error}
    end
  end

  @doc """
  Decompresses previously compressed data.
  """
  @spec decompress_data(binary()) :: {:ok, term()} | {:error, term()}
  def decompress_data(compressed_binary) do
    try do
      # Decompress using zlib
      decompressed_binary = :zlib.uncompress(compressed_binary)

      # Convert back to term
      data = :erlang.binary_to_term(decompressed_binary)

      {:ok, data}
    rescue
      error ->
        {:error, error}
    end
  end

  @doc """
  Checks if data is compressed (simple heuristic based on binary format).
  """
  @spec is_compressed?(binary()) :: boolean()
  def is_compressed?(data) when is_binary(data) do
    # Simple heuristic: compressed data typically starts with zlib header
    case data do
      <<0x78, 0x9C, _rest::binary>> -> true  # Default compression
      <<0x78, 0xDA, _rest::binary>> -> true  # Best compression
      <<0x78, 0x01, _rest::binary>> -> true  # No compression
      _ -> false
    end
  end
  def is_compressed?(_), do: false

  # Private Implementation

  defp find_compression_candidates(cutoff_time, access_threshold) do
    # Access tracking table for finding candidates
    access_table = :ast_repo_access_tracking

    try do
      case :ets.info(access_table, :size) do
        :undefined ->
          Logger.warning("Access tracking table not found, no compression candidates")
          []
        _ ->
          :ets.tab2list(access_table)
          |> Enum.filter(fn {_module, last_access, access_count} ->
            last_access < cutoff_time and access_count < access_threshold
          end)
          |> Enum.map(fn {module, _last_access, _access_count} -> module end)
      end
    rescue
      error ->
        Logger.warning("Failed to find compression candidates: #{inspect(error)}")
        []
    end
  end

  defp compress_candidates(candidates, compression_level) do
    Enum.reduce(candidates, {0, 0, 0}, fn module, {count, total_original, total_compressed} ->
      case compress_module_data(module, compression_level) do
        {:ok, original_size, compressed_size} ->
          {count + 1, total_original + original_size, total_compressed + compressed_size}
        {:error, reason} ->
          Logger.warning("Failed to compress module #{inspect(module)}: #{inspect(reason)}")
          {count, total_original, total_compressed}
      end
    end)
  end

  defp compress_module_data(module, compression_level) do
    try do
      # In a real implementation, this would fetch actual module data
      # from the Enhanced Repository and compress it

      # Simulate module data compression
      simulated_data = generate_simulated_module_data(module)

      case compress_data(simulated_data, compression_level) do
        {:ok, _compressed_binary, original_size, compressed_size} ->
          Logger.debug("Compressed module #{inspect(module)}: #{original_size} -> #{compressed_size} bytes")
          {:ok, original_size, compressed_size}
        {:error, reason} ->
          {:error, reason}
      end
    rescue
      error ->
        {:error, error}
    end
  end

  defp generate_simulated_module_data(module) do
    # Generate realistic simulated AST data for compression testing
    %{
      module: module,
      ast: generate_simulated_ast(),
      analysis: generate_simulated_analysis(),
      metadata: generate_simulated_metadata(),
      timestamp: System.monotonic_time(:millisecond)
    }
  end

  defp generate_simulated_ast() do
    # Simulate a complex AST structure
    functions = for i <- 1..10 do
      %{
        name: :"function_#{i}",
        arity: :rand.uniform(5),
        clauses: for j <- 1..3 do
          %{
            clause: j,
            params: (for k <- 1..:rand.uniform(3), do: :"param_#{k}"),
            body: generate_expression_tree(4)  # Depth 4 expression tree
          }
        end
      }
    end

    %{
      type: :module,
      name: :simulated_module,
      functions: functions,
      attributes: [
        {:module, :simulated_module},
        {:export, [{:test, 0}, {:run, 1}]}
      ]
    }
  end

  defp generate_expression_tree(0), do: {:atom, :leaf}
  defp generate_expression_tree(depth) do
    case :rand.uniform(4) do
      1 -> {:call, :rand.uniform(100), generate_expression_tree(depth - 1)}
      2 -> {:tuple, [generate_expression_tree(depth - 1), generate_expression_tree(depth - 1)]}
      3 -> {:list, for(_ <- 1..:rand.uniform(3), do: generate_expression_tree(depth - 1))}
      4 -> {:binary, :rand.bytes(32)}
    end
  end

  defp generate_simulated_analysis() do
    %{
      complexity: :rand.uniform(100),
      dependencies: for(_ <- 1..:rand.uniform(5), do: :"dep_#{:rand.uniform(1000)}"),
      type_info: %{
        functions: for i <- 1..5 do
          {:"func_#{i}", %{
            input_types: [:atom, :integer],
            output_type: :any,
            side_effects: :rand.uniform(2) == 1
          }}
        end
      },
      warnings: for(_ <- 1..:rand.uniform(3), do: "Warning #{:rand.uniform(100)}")
    }
  end

  defp generate_simulated_metadata() do
    %{
      file_path: "/fake/path/module.ex",
      line_count: :rand.uniform(500),
      last_modified: System.os_time(:second),
      checksum: :crypto.hash(:sha256, "fake_content") |> Base.encode16(),
      compile_options: [:debug_info, :warnings_as_errors]
    }
  end
end

1	defmodule ElixirScope.ASTRepository.MemoryManager.Compressor do
2	@moduledoc """
3	Data compression subsystem for compressing infrequently accessed analysis data.
4
5	Uses binary term compression to reduce memory footprint of large AST
6	structures and analysis results that are rarely accessed.
7	"""
8
9	require Logger
10
11	@type compression_stats :: %{
12	modules_compressed: non_neg_integer(),
13	compression_ratio: float(),
14	space_saved_bytes: non_neg_integer(),
15	last_compression_duration: non_neg_integer(),
16	total_compressions: non_neg_integer()
17	}
18
19	@doc """
20	Gets initial compression statistics structure.
21	"""
22	@spec get_initial_stats() :: compression_stats()
23	def get_initial_stats() do
24	%{	2✔
25	modules_compressed: 0,
26	compression_ratio: 0.0,
27	space_saved_bytes: 0,
28	last_compression_duration: 0,
29	total_compressions: 0
30	}
31	end
32
33	@doc """
34	Performs compression of infrequently accessed analysis data.
35
36	## Options
37
38	- `:access_threshold` - Minimum access count to avoid compression (default: 5)
39	- `:age_threshold` - Minimum age in seconds before compression (default: 1800)
40	- `:compression_level` - Compression level 1-9 (default: 6)
41	"""
42	@spec perform_compression(keyword()) :: {:ok, compression_stats()} \| {:error, term()}
43	def perform_compression(opts \\ []) do
44	access_threshold = Keyword.get(opts, :access_threshold, 5)	8✔
45	age_threshold = Keyword.get(opts, :age_threshold, 1800)	8✔
46	compression_level = Keyword.get(opts, :compression_level, 6)	8✔
47
48	# Validate compression level
49	compression_level = case compression_level do	8✔
50	level when is_integer(level) and level >= 1 and level <= 9 -> level	8✔
51	_ -> 6 # Default to level 6 if invalid	×
52	end
53
54	current_time = System.monotonic_time(:second)	8✔
55	cutoff_time = current_time - age_threshold	8✔
56
57	try do	8✔
58	# Find data to compress
59	candidates = find_compression_candidates(cutoff_time, access_threshold)	8✔
60
61	# Perform compression
62	{compressed_count, total_original, total_compressed} = compress_candidates(candidates, compression_level)	8✔
63
64	compression_ratio = if total_original > 0 do	8✔
UNCOV 65	total_compressed / total_original	×
66	else
67	0.0
68	end
69
70	space_saved = total_original - total_compressed	8✔
71
72	result = %{	8✔
73	modules_compressed: compressed_count,
74	compression_ratio: compression_ratio,
75	space_saved_bytes: space_saved
76	}
77
78	Logger.info("Compression completed: #{compressed_count} modules, #{Float.round(compression_ratio * 100, 1)}% of original size")	8✔
79
80	{:ok, result}
81	rescue
82	error ->	×
83	Logger.error("Compression failed: #{inspect(error)}")	×
84	{:error, {:compression_failed, error}}
85	end
86	end
87
88	@doc """
89	Updates compression statistics with new results.
90	"""
91	@spec update_stats(compression_stats(), compression_stats(), non_neg_integer()) :: compression_stats()
92	def update_stats(stats, result, duration) do
93	%{stats \|	3✔
94	modules_compressed: stats.modules_compressed + result.modules_compressed,	3✔
95	compression_ratio: result.compression_ratio,	3✔
96	space_saved_bytes: stats.space_saved_bytes + result.space_saved_bytes,	3✔
97	last_compression_duration: duration,
98	total_compressions: stats.total_compressions + 1	3✔
99	}
100	end
101
102	@doc """
103	Compresses a single data structure using the specified compression level.
104	"""
105	@spec compress_data(term(), integer()) :: {:ok, binary(), non_neg_integer(), non_neg_integer()} \| {:error, term()}
106	def compress_data(data, compression_level \\ 6) do
107	try do	2✔
108	# Convert to binary term format
109	original_binary = :erlang.term_to_binary(data)	2✔
110	original_size = byte_size(original_binary)	2✔
111
112	# Compress using zlib
113	compressed_binary = :zlib.compress(original_binary)	2✔
114	compressed_size = byte_size(compressed_binary)	2✔
115
116	{:ok, compressed_binary, original_size, compressed_size}	2✔
117	rescue
118	error ->	×
119	{:error, error}
120	end
121	end
122
123	@doc """
124	Decompresses previously compressed data.
125	"""
126	@spec decompress_data(binary()) :: {:ok, term()} \| {:error, term()}
127	def decompress_data(compressed_binary) do
128	try do	1✔
129	# Decompress using zlib
130	decompressed_binary = :zlib.uncompress(compressed_binary)	1✔
131
132	# Convert back to term
133	data = :erlang.binary_to_term(decompressed_binary)	1✔
134
135	{:ok, data}
136	rescue
137	error ->	×
138	{:error, error}
139	end
140	end
141
142	@doc """
143	Checks if data is compressed (simple heuristic based on binary format).
144	"""
145	@spec is_compressed?(binary()) :: boolean()
146	def is_compressed?(data) when is_binary(data) do
147	# Simple heuristic: compressed data typically starts with zlib header
148	case data do	2✔
149	<<0x78, 0x9C, _rest::binary>> -> true # Default compression	1✔
150	<<0x78, 0xDA, _rest::binary>> -> true # Best compression	×
151	<<0x78, 0x01, _rest::binary>> -> true # No compression	×
152	_ -> false	1✔
153	end
154	end
155	def is_compressed?(_), do: false	×
156
157	# Private Implementation
158
159	defp find_compression_candidates(cutoff_time, access_threshold) do
160	# Access tracking table for finding candidates
161	access_table = :ast_repo_access_tracking	8✔
162
163	try do	8✔
164	case :ets.info(access_table, :size) do	8✔
165	:undefined ->
166	Logger.warning("Access tracking table not found, no compression candidates")	4✔
167	[]
168	_ ->
169	:ets.tab2list(access_table)
170	\|> Enum.filter(fn {_module, last_access, access_count} ->
171	last_access < cutoff_time and access_count < access_threshold	80✔
172	end)
173	\|> Enum.map(fn {module, _last_access, _access_count} -> module end)	4✔
174	end
175	rescue
176	error ->	×
177	Logger.warning("Failed to find compression candidates: #{inspect(error)}")	×
178	[]
179	end
180	end
181
182	defp compress_candidates(candidates, compression_level) do
183	Enum.reduce(candidates, {0, 0, 0}, fn module, {count, total_original, total_compressed} ->	8✔
UNCOV 184	case compress_module_data(module, compression_level) do	×
185	{:ok, original_size, compressed_size} ->
UNCOV 186	{count + 1, total_original + original_size, total_compressed + compressed_size}	×
187	{:error, reason} ->
188	Logger.warning("Failed to compress module #{inspect(module)}: #{inspect(reason)}")	×
189	{count, total_original, total_compressed}	×
190	end
191	end)
192	end
193
194	defp compress_module_data(module, compression_level) do
UNCOV 195	try do	×
196	# In a real implementation, this would fetch actual module data
197	# from the Enhanced Repository and compress it
198
199	# Simulate module data compression
UNCOV 200	simulated_data = generate_simulated_module_data(module)	×
201
UNCOV 202	case compress_data(simulated_data, compression_level) do	×
203	{:ok, _compressed_binary, original_size, compressed_size} ->
UNCOV 204	Logger.debug("Compressed module #{inspect(module)}: #{original_size} -> #{compressed_size} bytes")	×
UNCOV 205	{:ok, original_size, compressed_size}	×
206	{:error, reason} ->	×
207	{:error, reason}
208	end
209	rescue
210	error ->	×
211	{:error, error}
212	end
213	end
214
215	defp generate_simulated_module_data(module) do
216	# Generate realistic simulated AST data for compression testing
UNCOV 217	%{	×
218	module: module,
219	ast: generate_simulated_ast(),
220	analysis: generate_simulated_analysis(),
221	metadata: generate_simulated_metadata(),
222	timestamp: System.monotonic_time(:millisecond)
223	}
224	end
225
226	defp generate_simulated_ast() do
227	# Simulate a complex AST structure
UNCOV 228	functions = for i <- 1..10 do	×
UNCOV 229	%{	×
UNCOV 230	name: :"function_#{i}",	×
231	arity: :rand.uniform(5),
232	clauses: for j <- 1..3 do
UNCOV 233	%{	×
234	clause: j,
UNCOV 235	params: (for k <- 1..:rand.uniform(3), do: :"param_#{k}"),	×
236	body: generate_expression_tree(4) # Depth 4 expression tree
237	}
238	end
239	}
240	end
241
UNCOV 242	%{	×
243	type: :module,
244	name: :simulated_module,
245	functions: functions,
246	attributes: [
247	{:module, :simulated_module},
248	{:export, [{:test, 0}, {:run, 1}]}
249	]
250	}
251	end
252
UNCOV 253	defp generate_expression_tree(0), do: {:atom, :leaf}	×
254	defp generate_expression_tree(depth) do
UNCOV 255	case :rand.uniform(4) do	×
UNCOV 256	1 -> {:call, :rand.uniform(100), generate_expression_tree(depth - 1)}	×
UNCOV 257	2 -> {:tuple, [generate_expression_tree(depth - 1), generate_expression_tree(depth - 1)]}	×
UNCOV 258	3 -> {:list, for(_ <- 1..:rand.uniform(3), do: generate_expression_tree(depth - 1))}	×
UNCOV 259	4 -> {:binary, :rand.bytes(32)}	×
260	end
261	end
262
263	defp generate_simulated_analysis() do
UNCOV 264	%{	×
265	complexity: :rand.uniform(100),
UNCOV 266	dependencies: for(_ <- 1..:rand.uniform(5), do: :"dep_#{:rand.uniform(1000)}"),	×
267	type_info: %{
UNCOV 268	functions: for i <- 1..5 do	×
UNCOV 269	{:"func_#{i}", %{	×
270	input_types: [:atom, :integer],
271	output_type: :any,
272	side_effects: :rand.uniform(2) == 1
273	}}
274	end
275	},
UNCOV 276	warnings: for(_ <- 1..:rand.uniform(3), do: "Warning #{:rand.uniform(100)}")	×
277	}
278	end
279
280	defp generate_simulated_metadata() do
UNCOV 281	%{	×
282	file_path: "/fake/path/module.ex",
283	line_count: :rand.uniform(500),
284	last_modified: System.os_time(:second),
285	checksum: :crypto.hash(:sha256, "fake_content") \|> Base.encode16(),
286	compile_options: [:debug_info, :warnings_as_errors]
287	}
288	end
289	end

nshkrdotcom / ElixirScope / c7479091597c3382f92c87a4fd0395db3ee30193

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous