9c2823cf899c9314fd4ccec9dd6c3b589d83e839

Committed 04 Dec 2025 05:47PM UTC coverage: 88.716% (-0.9%) from 89.621%

Build # 9c2823cf899c9314fd4ccec9dd6c3b589d83e839

Build Type

push

github

Committed by

web-flow

Commit Message

AI-powered marking (#1248)

* feat: v1 of AI-generated comments

* feat: added logging of inputs and outputs

* Update generate_ai_comments.ex

* feat: function to save outputs to database

* Format answers json before sending to LLM

* Add LLM Prompt to question params when submitting assessment xml file

* Add LLM Prompt to api response when grading view is open

* feat: added llm_prompt from qn to raw_prompt

* feat: enabling/disabling of LLM feature by course level

* feat: added llm_grading boolean field to course creation API

* feat: added api key storage in courses & edit api key/enable llm grading

* feat: encryption for llm_api_key

* feat: added final comment editing route

* feat: added logging of chosen comments

* fix: bugs when certain fields were missing

* feat: updated tests

* formatting

* fix: error handling when calling openai API

* fix: credo issues

* formatting

* Address some comments

* Fix formatting

* rm IO.inspect

* a

* Use case instead of if

* Streamlines generate_ai_comments to only send the selected question and its relevant info + use the correct llm_prompt

* Remove unncessary field

* default: false for llm_grading

* Add proper linking between ai_comments table and submissions. Return it to submission retrieval as well

* Resolve some migration comments

* Add llm_model and llm_api_url to the DB + schema

* Moves api key, api url, llm model and course prompt to course level

* Add encryption_key to env

* Do not hardcode formatting instructions

* Add Assessment level prompts to the XML

* Return some additional info for composing of prompts

* Remove un-used 'save comments'

* Fix existing assessment tests

* Fix generate_ai_comments test cases

* Fix bug preventing avengers from generating ai comments

* Fix up tests + error msgs

* Formatting

* some mix credo suggestions

* format

* Fix credo issue

* bug fix + credo fixes

* Fix tests

* format

* Modify test.exs

* Update lib/cadet_web/controllers/gener... (continued)

Run Details

118 of 174 new or added lines in 9 files covered. (67.82%)

1 existing line in 1 file now uncovered.

3758 of 4236 relevant lines covered (88.72%)

7103.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.43

/lib/cadet/courses/course.ex

defmodule Cadet.Courses.Course do
  @moduledoc """
  The Course entity stores the configuration of a particular course.
  """
  use Cadet, :model

  alias Cadet.Courses.AssessmentConfig
  alias CadetWeb.AICommentsHelpers

  @type t :: %__MODULE__{
          course_name: String.t(),
          course_short_name: String.t(),
          viewable: boolean(),
          enable_game: boolean(),
          enable_achievements: boolean(),
          enable_overall_leaderboard: boolean(),
          enable_contest_leaderboard: boolean(),
          top_leaderboard_display: integer(),
          top_contest_leaderboard_display: integer(),
          enable_sourcecast: boolean(),
          enable_stories: boolean(),
          enable_llm_grading: boolean(),
          llm_api_key: String.t() | nil,
          llm_model: String.t() | nil,
          llm_api_url: String.t() | nil,
          llm_course_level_prompt: String.t() | nil,
          source_chapter: integer(),
          source_variant: String.t(),
          module_help_text: String.t(),
          assets_prefix: String.t() | nil
        }

  schema "courses" do
    field(:course_name, :string)
    field(:course_short_name, :string)
    field(:viewable, :boolean, default: true)
    field(:enable_game, :boolean, default: true)
    field(:enable_achievements, :boolean, default: true)
    field(:enable_overall_leaderboard, :boolean, default: true)
    field(:enable_contest_leaderboard, :boolean, default: true)
    field(:top_leaderboard_display, :integer, default: 100)
    field(:top_contest_leaderboard_display, :integer, default: 10)
    field(:enable_sourcecast, :boolean, default: true)
    field(:enable_stories, :boolean, default: false)
    field(:enable_llm_grading, :boolean, default: false)
    field(:llm_api_key, :string, default: nil)
    field(:llm_model, :string, default: nil)
    field(:llm_api_url, :string, default: nil)
    field(:llm_course_level_prompt, :string, default: nil)
    field(:source_chapter, :integer)
    field(:source_variant, :string)
    field(:module_help_text, :string)

    # for now, only settable from database
    field(:assets_prefix, :string, default: nil)

    has_many(:assessment_config, AssessmentConfig)

    timestamps()
  end

  @required_fields ~w(course_name viewable enable_game
    enable_achievements enable_overall_leaderboard enable_contest_leaderboard top_leaderboard_display top_contest_leaderboard_display enable_sourcecast enable_stories source_chapter source_variant)a
  @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key llm_model llm_api_url llm_course_level_prompt)a

  def changeset(course, params) do
    course
    |> cast(params, @required_fields ++ @optional_fields)
    |> validate_required(@required_fields)
    |> validate_sublanguage_combination(params)
    |> put_encrypted_llm_api_key()
  end

  def put_encrypted_llm_api_key(changeset) do
    if llm_api_key = get_change(changeset, :llm_api_key) do
      if is_binary(llm_api_key) and llm_api_key != "" do
        encrypted = AICommentsHelpers.encrypt_llm_api_key(llm_api_key)

        case encrypted do
          {:error, :invalid_encryption_key} ->
            add_error(
              changeset,
              :llm_api_key,
              "encryption key is not configured properly, cannot store LLM API key"
            )

          encrypted ->
            put_change(changeset, :llm_api_key, encrypted)
        end
      else
        # If empty string or nil is provided, don't encrypt but don't add error
        changeset
      end
    else
      changeset
    end
  end

  # Validates combination of Source chapter and variant
  defp validate_sublanguage_combination(changeset, params) do
    chap = Map.has_key?(params, :source_chapter)
    var = Map.has_key?(params, :source_variant)

    # not (chap xor var)
    case {chap, var} do
      {true, true} ->
        case get_field(changeset, :source_chapter) do
          1 ->
            validate_inclusion(changeset, :source_variant, [
              "default",
              "lazy",
              "wasm",
              "native",
              "typed"
            ])

          2 ->
            validate_inclusion(changeset, :source_variant, ["default", "lazy", "native", "typed"])

          3 ->
            validate_inclusion(changeset, :source_variant, [
              "default",
              "concurrent",
              "non-det",
              "native",
              "typed"
            ])

          4 ->
            validate_inclusion(changeset, :source_variant, ["default", "gpu", "native"])

          _ ->
            add_error(changeset, :source_chapter, "is invalid")
        end

      {false, false} ->
        changeset

      {_, _} ->
        add_error(
          changeset,
          :source_chapter,
          "source chapter and source variant must be present together"
        )
    end
  end
end

1	defmodule Cadet.Courses.Course do
2	@moduledoc """
3	The Course entity stores the configuration of a particular course.
4	"""
5	use Cadet, :model
6
7	alias Cadet.Courses.AssessmentConfig
8	alias CadetWeb.AICommentsHelpers
9
10	@type t :: %__MODULE__{
11	course_name: String.t(),
12	course_short_name: String.t(),
13	viewable: boolean(),
14	enable_game: boolean(),
15	enable_achievements: boolean(),
16	enable_overall_leaderboard: boolean(),
17	enable_contest_leaderboard: boolean(),
18	top_leaderboard_display: integer(),
19	top_contest_leaderboard_display: integer(),
20	enable_sourcecast: boolean(),
21	enable_stories: boolean(),
22	enable_llm_grading: boolean(),
23	llm_api_key: String.t() \| nil,
24	llm_model: String.t() \| nil,
25	llm_api_url: String.t() \| nil,
26	llm_course_level_prompt: String.t() \| nil,
27	source_chapter: integer(),
28	source_variant: String.t(),
29	module_help_text: String.t(),
30	assets_prefix: String.t() \| nil
31	}
32
33	schema "courses" do	14,800,469✔
34	field(:course_name, :string)
35	field(:course_short_name, :string)
36	field(:viewable, :boolean, default: true)
37	field(:enable_game, :boolean, default: true)
38	field(:enable_achievements, :boolean, default: true)
39	field(:enable_overall_leaderboard, :boolean, default: true)
40	field(:enable_contest_leaderboard, :boolean, default: true)
41	field(:top_leaderboard_display, :integer, default: 100)
42	field(:top_contest_leaderboard_display, :integer, default: 10)
43	field(:enable_sourcecast, :boolean, default: true)
44	field(:enable_stories, :boolean, default: false)
45	field(:enable_llm_grading, :boolean, default: false)
46	field(:llm_api_key, :string, default: nil)
47	field(:llm_model, :string, default: nil)
48	field(:llm_api_url, :string, default: nil)
49	field(:llm_course_level_prompt, :string, default: nil)
50	field(:source_chapter, :integer)
51	field(:source_variant, :string)
52	field(:module_help_text, :string)
53
54	# for now, only settable from database
55	field(:assets_prefix, :string, default: nil)
56
57	has_many(:assessment_config, AssessmentConfig)
58
59	timestamps()
60	end
61
62	@required_fields ~w(course_name viewable enable_game
63	enable_achievements enable_overall_leaderboard enable_contest_leaderboard top_leaderboard_display top_contest_leaderboard_display enable_sourcecast enable_stories source_chapter source_variant)a
64	@optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key llm_model llm_api_url llm_course_level_prompt)a
65
66	def changeset(course, params) do
67	course
68	\|> cast(params, @required_fields ++ @optional_fields)
69	\|> validate_required(@required_fields)
70	\|> validate_sublanguage_combination(params)
71	\|> put_encrypted_llm_api_key()	38✔
72	end
73
74	def put_encrypted_llm_api_key(changeset) do
75	if llm_api_key = get_change(changeset, :llm_api_key) do	38✔
NEW 76	if is_binary(llm_api_key) and llm_api_key != "" do	×
NEW 77	encrypted = AICommentsHelpers.encrypt_llm_api_key(llm_api_key)	×
78
NEW 79	case encrypted do	×
80	{:error, :invalid_encryption_key} ->
NEW 81	add_error(	×
82	changeset,
83	:llm_api_key,
84	"encryption key is not configured properly, cannot store LLM API key"
85	)
86
87	encrypted ->
NEW 88	put_change(changeset, :llm_api_key, encrypted)	×
89	end
90	else
91	# If empty string or nil is provided, don't encrypt but don't add error
NEW 92	changeset	×
93	end
94	else
95	changeset	38✔
96	end
97	end
98
99	# Validates combination of Source chapter and variant
100	defp validate_sublanguage_combination(changeset, params) do
101	chap = Map.has_key?(params, :source_chapter)	38✔
102	var = Map.has_key?(params, :source_variant)	38✔
103
104	# not (chap xor var)
105	case {chap, var} do	38✔
106	{true, true} ->
107	case get_field(changeset, :source_chapter) do	31✔
108	1 ->
109	validate_inclusion(changeset, :source_variant, [	18✔
110	"default",
111	"lazy",
112	"wasm",
113	"native",
114	"typed"
115	])
116
117	2 ->
118	validate_inclusion(changeset, :source_variant, ["default", "lazy", "native", "typed"])	4✔
119
120	3 ->
121	validate_inclusion(changeset, :source_variant, [	2✔
122	"default",
123	"concurrent",
124	"non-det",
125	"native",
126	"typed"
127	])
128
129	4 ->
130	validate_inclusion(changeset, :source_variant, ["default", "gpu", "native"])	5✔
131
132	_ ->
133	add_error(changeset, :source_chapter, "is invalid")	2✔
134	end
135
136	{false, false} ->
137	changeset	3✔
138
139	{_, _} ->
140	add_error(	4✔
141	changeset,
142	:source_chapter,
143	"source chapter and source variant must be present together"
144	)
145	end
146	end
147	end

source-academy / backend / 9c2823cf899c9314fd4ccec9dd6c3b589d83e839

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous