e0330f2cf38b2d8af12bffd20f4cac2158d607fc-PR-1236

Committed 31 Mar 2025 09:12AM UTC coverage: 19.982% (-73.6%) from 93.607%

Build # e0330f2cf38b2d8af12bffd20f4cac2158d607fc-PR-1236

Build Type

Pull #1236

github

Committed by

RichDom2185

Commit Message

Redate migrations to maintain total ordering

Pull Request Pull Request #1236: Added Exam mode

Run Details

12 of 57 new or added lines in 8 files covered. (21.05%)

2430 existing lines in 97 files now uncovered.

671 of 3358 relevant lines covered (19.98%)

3.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/lib/cadet/jobs/autograder/lambda_worker.ex

defmodule Cadet.Autograder.LambdaWorker do
  # Suppress no_match from macro
  @dialyzer {:no_match, __after_compile__: 2}
  @moduledoc """
  This module submits the answer to the autograder and creates a job for the ResultStoreWorker to
  write the received result to db.
  """
  use Que.Worker, concurrency: 20

  require Logger

  alias Cadet.Autograder.ResultStoreWorker
  alias Cadet.Assessments.{Answer, Question}

  @doc """
  This Que callback transforms an input of %{question: %Question{}, answer: %Answer{}} into
  the correct shape to dispatch to lambda, waits for the response, parses it, and enqueues a
  storage job.
  """
  def perform(params = %{answer: answer = %Answer{}, question: %Question{}}) do
    lambda_params = build_request_params(params)

    if Enum.empty?(lambda_params.testcases) do
      Logger.warn("No testcases found. Skipping autograding for answer_id: #{answer.id}")
      # Fix for https://github.com/source-academy/backend/issues/472
      Process.sleep(1000)
    else
      response =
        :cadet
        |> Application.fetch_env!(:autograder)
        |> Keyword.get(:lambda_name)
        |> ExAws.Lambda.invoke(lambda_params, %{})
        |> ExAws.request!()

      result = parse_response(response)

      Que.add(ResultStoreWorker, %{
        answer_id: answer.id,
        result: result,
        overwrite: params[:overwrite] || false
      })
    end
  end

  def on_failure(%{answer: answer = %Answer{}, question: %Question{}}, error) do
    error_message =
      "Failed to get autograder result. answer_id: #{answer.id}, error: #{inspect(error, pretty: true)}"

    Logger.error(error_message)
    Sentry.capture_message(error_message)

    Que.add(
      ResultStoreWorker,
      %{
        answer_id: answer.id,
        result: %{
          score: 0,
          max_score: 1,
          status: :failed,
          result: [
            %{
              "resultType" => "error",
              "errors" => [
                %{
                  "errorType" => "systemError",
                  "errorMessage" =>
                    "Autograder runtime error. Please contact a system administrator"
                }
              ]
            }
          ]
        }
      }
    )
  end

  def build_request_params(%{question: question = %Question{}, answer: answer = %Answer{}}) do
    question_content = question.question

    {_, upcased_name_external} =
      question.grading_library.external
      |> Map.from_struct()
      |> Map.get_and_update(
        :name,
        &{&1, &1 |> String.upcase()}
      )

    %{
      prependProgram: Map.get(question_content, "prepend", ""),
      studentProgram: Map.get(answer.answer, "code"),
      postpendProgram: Map.get(question_content, "postpend", ""),
      testcases:
        Map.get(question_content, "public", []) ++
          Map.get(question_content, "opaque", []) ++ Map.get(question_content, "secret", []),
      library: %{
        chapter: question.grading_library.chapter,
        external: upcased_name_external,
        globals: Enum.map(question.grading_library.globals, fn {k, v} -> [k, v] end)
      }
    }
  end

  defp parse_response(response) when is_map(response) do
    # If the lambda crashes, results are in the format of:
    # %{"errorMessage" => "${message}"}
    if Map.has_key?(response, "errorMessage") do
      %{
        score: 0,
        max_score: 1,
        status: :failed,
        result: [
          %{
            "resultType" => "error",
            "errors" => [
              %{"errorType" => "systemError", "errorMessage" => response["errorMessage"]}
            ]
          }
        ]
      }
    else
      %{
        score: response["totalScore"],
        max_score: response["maxScore"],
        result: response["results"],
        status: :success
      }
    end
  end
end

1	defmodule Cadet.Autograder.LambdaWorker do
2	# Suppress no_match from macro
3	@dialyzer {:no_match, __after_compile__: 2}
4	@moduledoc """
5	This module submits the answer to the autograder and creates a job for the ResultStoreWorker to
6	write the received result to db.
7	"""
8	use Que.Worker, concurrency: 20
9
10	require Logger
11
12	alias Cadet.Autograder.ResultStoreWorker
13	alias Cadet.Assessments.{Answer, Question}
14
15	@doc """
16	This Que callback transforms an input of %{question: %Question{}, answer: %Answer{}} into
17	the correct shape to dispatch to lambda, waits for the response, parses it, and enqueues a
18	storage job.
19	"""
20	def perform(params = %{answer: answer = %Answer{}, question: %Question{}}) do
UNCOV 21	lambda_params = build_request_params(params)	×
22
UNCOV 23	if Enum.empty?(lambda_params.testcases) do	×
UNCOV 24	Logger.warn("No testcases found. Skipping autograding for answer_id: #{answer.id}")	×
25	# Fix for https://github.com/source-academy/backend/issues/472
UNCOV 26	Process.sleep(1000)	×
27	else
UNCOV 28	response =	×
29	:cadet
30	\|> Application.fetch_env!(:autograder)
31	\|> Keyword.get(:lambda_name)
32	\|> ExAws.Lambda.invoke(lambda_params, %{})
33	\|> ExAws.request!()
34
UNCOV 35	result = parse_response(response)	×
36
UNCOV 37	Que.add(ResultStoreWorker, %{	×
UNCOV 38	answer_id: answer.id,	×
39	result: result,
UNCOV 40	overwrite: params[:overwrite] \|\| false	×
41	})
42	end
43	end
44
45	def on_failure(%{answer: answer = %Answer{}, question: %Question{}}, error) do
UNCOV 46	error_message =	×
UNCOV 47	"Failed to get autograder result. answer_id: #{answer.id}, error: #{inspect(error, pretty: true)}"	×
48
UNCOV 49	Logger.error(error_message)	×
UNCOV 50	Sentry.capture_message(error_message)	×
51
UNCOV 52	Que.add(	×
53	ResultStoreWorker,
54	%{
UNCOV 55	answer_id: answer.id,	×
56	result: %{
57	score: 0,
58	max_score: 1,
59	status: :failed,
60	result: [
61	%{
62	"resultType" => "error",
63	"errors" => [
64	%{
65	"errorType" => "systemError",
66	"errorMessage" =>
67	"Autograder runtime error. Please contact a system administrator"
68	}
69	]
70	}
71	]
72	}
73	}
74	)
75	end
76
77	def build_request_params(%{question: question = %Question{}, answer: answer = %Answer{}}) do
UNCOV 78	question_content = question.question	×
79
UNCOV 80	{_, upcased_name_external} =	×
UNCOV 81	question.grading_library.external	×
82	\|> Map.from_struct()
83	\|> Map.get_and_update(
84	:name,
UNCOV 85	&{&1, &1 \|> String.upcase()}	×
86	)
87
UNCOV 88	%{	×
89	prependProgram: Map.get(question_content, "prepend", ""),
UNCOV 90	studentProgram: Map.get(answer.answer, "code"),	×
91	postpendProgram: Map.get(question_content, "postpend", ""),
92	testcases:
93	Map.get(question_content, "public", []) ++
94	Map.get(question_content, "opaque", []) ++ Map.get(question_content, "secret", []),
95	library: %{
UNCOV 96	chapter: question.grading_library.chapter,	×
97	external: upcased_name_external,
UNCOV 98	globals: Enum.map(question.grading_library.globals, fn {k, v} -> [k, v] end)	×
99	}
100	}
101	end
102
103	defp parse_response(response) when is_map(response) do
104	# If the lambda crashes, results are in the format of:
105	# %{"errorMessage" => "${message}"}
UNCOV 106	if Map.has_key?(response, "errorMessage") do	×
UNCOV 107	%{	×
108	score: 0,
109	max_score: 1,
110	status: :failed,
111	result: [
112	%{
113	"resultType" => "error",
114	"errors" => [
115	%{"errorType" => "systemError", "errorMessage" => response["errorMessage"]}
116	]
117	}
118	]
119	}
120	else
UNCOV 121	%{	×
122	score: response["totalScore"],
123	max_score: response["maxScore"],
124	result: response["results"],
125	status: :success
126	}
127	end
128	end
129	end

source-academy / backend / e0330f2cf38b2d8af12bffd20f4cac2158d607fc-PR-1236

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous