28600002640

Committed 02 Jul 2026 03:00PM UTC coverage: 73.516% (-0.009%) from 73.525%

Build # 28600002640

Build Type

push

github

Committed by

web-flow

Commit Message

Add CONTENTdm sync support for document sets (#5649)

Coverage Stats

2691 of 4245 branches covered (63.39%)

Branch coverage included in aggregate %.

5 of 15 new or added lines in 1 file covered. (33.33%)

1 existing line in 1 file now uncovered.

10897 of 14238 relevant lines covered (76.53%)

195.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.08

/app/controllers/export_controller.rb

require 'contentdm_translator'

class ExportController < ApplicationController
  require 'zip'

  include CollectionHelper
  include ExportHelper
  include ExportService

  DEFAULT_WORKS_PER_PAGE = 15

  before_action :require_owner, only: [:index]

  def index
    filtered_data

    respond_to do |format|
      format.html
      format.turbo_stream
    end
  end

  def show
    xhtml = work_to_xhtml(@work)

    render text: xhtml, layout: false
  end

  def printable
    result = Work::Export::Printable.new(
      work: @work,
      format: params[:format],
      edition: params[:edition],
      include_metadata: true,
      include_contributors: true,
      include_notes: false,
      preserve_lb: false
    ).call

    if result.success?
      send_data(
        File.read(result.file),
        filename: result.filename,
        content_type: result.content_type
      )

      cookies['download_finished'] = 'true'
    else
      head :internal_server_error
    end
  end

  def tei
    tei_xml = work_to_tei(@work, current_user)

    render text: tei_xml, content_type: 'application/xml', layout: false
  end

  def subject_details_csv
    send_data(
      @collection.export_subject_details_as_csv,
      filename: "fromthepage_subject_details_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
      type: 'application/csv'
    )
    cookies['download_finished'] = 'true'
  end

  def subject_coocurrence_csv
    send_data(
      @collection.export_subject_coocurrence_as_csv,
      filename: "fromthepage_subject_coocurrence_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
      type: 'application/csv'
    )
    cookies['download_finished'] = 'true'
  end

  def subject_distribution_csv
    send_data(
      @collection.export_subject_distribution_as_csv(@article),
      filename: "fromthepage_subject_distribution_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
      type: 'application/csv'
    )
    cookies['download_finished'] = 'true'
  end

  def subject_index_csv
    send_data(
      @collection.export_subject_index_as_csv(@collection.works),
      filename: "fromthepage_subject_index_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
      type: 'application/csv'
    )
    cookies['download_finished'] = 'true'
  end

  def work_metadata_csv
    filename = params[:filename] ? "#{params[:filename]}.csv" : "fromthepage_work_metadata_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv"
    result = Work::Metadata::ExportCsv.new(collection: @collection, works: @collection.works).call

    send_data(
      result.csv_string,
      filename: filename,
      type: 'application/csv'
    )
    cookies['download_finished'] = 'true'
  end

  def table_csv
    filename = "fromthepage_tables_export_#{@work.id}_#{Time.now.utc.iso8601}.csv"
    collection = @work.collection

    if collection.field_based?
      result = Work::Table::ExportCsv.new(
        collection: collection,
        work_ids: [@work.id]
      ).call

      raise 'Failed to export csv' unless result.success?

      csv_string = result.csv_string
    else
      csv_string = export_tables_as_csv(@work)
    end

    send_data(
      csv_string,
      filename: filename,
      type: 'text/csv'
    )
    cookies['download_finished'] = 'true'
  end

  def export_all_tables
    filename = "fromthepage_tables_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv"

    if @collection.field_based?
      result = Work::Table::ExportCsv.new(
        collection: @collection,
        work_ids: @collection.works.pluck(:id)
      ).call

      csv_string = result.csv_string
    else
      csv_string = export_tables_as_csv(@collection)
    end
    send_data(
      csv_string,
      filename: filename,
      type: 'application/csv'
    )
    cookies['download_finished'] = 'true'
  end

  def page_plaintext_verbatim
    render  layout: false, content_type: 'text/plain', plain: @page.verbatim_transcription_plaintext
  end

  def page_plaintext_translation_verbatim
    render  layout: false, content_type: 'text/plain', plain: @page.verbatim_translation_plaintext
  end

  def page_plaintext_emended
    render  layout: false, content_type: 'text/plain', plain: @page.emended_transcription_plaintext
  end

  def page_plaintext_translation_emended
    render  layout: false, content_type: 'text/plain', plain: @page.emended_translation_plaintext
  end

  def page_plaintext_searchable
    render  layout: false, content_type: 'text/plain', plain: @page.search_text
  end

  def work_plaintext_verbatim
    render  layout: false, content_type: 'text/plain', plain: @work.verbatim_transcription_plaintext
  end

  def work_plaintext_translation_verbatim
    render  layout: false, content_type: 'text/plain', plain: @work.verbatim_translation_plaintext
  end

  def work_plaintext_emended
    render  layout: false, content_type: 'text/plain', plain: @work.emended_transcription_plaintext
  end

  def work_plaintext_translation_emended
    render  layout: false, content_type: 'text/plain', plain: @work.emended_translation_plaintext
  end

  def work_plaintext_searchable
    render  layout: false, content_type: 'text/plain', plain: @work.searchable_plaintext
  end

  def edit_contentdm_credentials
    @sync_target = @collection
    @cdm_collection = cdm_collection_for(@sync_target)

    if ContentdmTranslator.collection_is_cdm?(@sync_target)
      begin
        field_config = ContentdmTranslator.fetch_cdm_field_config(@sync_target)
        @cdm_fulltext_fields = field_config.select { |e| e['type'] == 'FTS' }.map { |e| [e['name'], e['nick']] }
        @cdm_metadata_fields = field_config.map { |e| [e['name'], e['nick']] }
      rescue => e
        Rails.logger.error("Failed to fetch CONTENTdm field config: #{e.message}")
      end
    end
  end

  # TODO: Add specs for this
  def update_contentdm_credentials
    # test credentials
    license_key = params[:collection][:license_key]
    contentdm_user_name = params[:contentdm_user_name]
    contentdm_password = params[:contentdm_password]
    error_message, _fts_field = ContentdmTranslator.fts_field_for_collection(@collection)

    # persist license key and export settings so the user doesn't have to retype them
    if error_message.blank? || !error_message.match(/license.*invalid/)
      cdm_collection = cdm_collection_for(@collection)
      cdm_collection.license_key = license_key
      cdm_collection.save!

      cdm_setting = cdm_collection.cdm_export_setting || cdm_collection.build_cdm_export_setting
      cdm_setting.transcript_source    = params[:transcript_source].presence || CdmExportSetting::HUMAN_ONLY
      cdm_setting.fulltext_field       = params[:cdm_fulltext_field].presence
      cdm_setting.include_ai_provenance = params[:include_ai_provenance] == '1'
      cdm_setting.ai_provenance_field  = params[:cdm_ai_provenance_field].presence
      cdm_setting.prepend_ai_warning   = params[:prepend_ai_warning] == '1'
      cdm_setting.save!
    end

    # redirect to or render edit screen with error
    if error_message
      flash[:error] = error_message
      @sync_target = @collection
      @cdm_collection = cdm_collection_for(@sync_target)
      render action: :edit_contentdm_credentials, collection_id: @collection.slug
      return
    end

    # pass credentials, FTS field, and search to background job
    log_file = ContentdmTranslator.log_file(@collection)
    FileUtils.mkdir_p(File.dirname(log_file)) unless Dir.exist? File.dirname(log_file)
    cmd = "rake fromthepage:cdm_transcript_export[#{@collection.slug}] > #{log_file} 2>&1 &"
    logger.info(cmd)
    system({ 'contentdm_username' => contentdm_user_name, 'contentdm_password' => contentdm_password, 'contentdm_license' => license_key }, cmd)

    # display results somehow
    flash[:notice] = t('.updating_contentdm_message')
    ajax_redirect_to action: :index, collection_id: @collection.slug
  end

  private

  def cdm_collection_for(collection_or_set)
    collection_or_set.is_a?(DocumentSet) ? collection_or_set.collection : collection_or_set
  end

  def require_owner
    unless user_signed_in? && current_user.like_owner?(@collection)
      redirect_to main_app.dashboard_path
    end
  end

  def filtered_data
    @sorting = (params[:sort] || 'title').to_sym
    @ordering = (params[:order] || 'ASC').downcase.to_sym
    @ordering = [:asc, :desc].include?(@ordering) ? @ordering : :desc

    # Check if there are any translated works in the collection
    @header = @collection.works.where(supports_translation: true).exists? ? 'Translated' : 'Transcribed'

    @works = params[:search].blank? ? @collection.works : @collection.search_works(params[:search])
    @works = @works.includes(:work_statistic)

    @work_stats_hash_map = {}
    @works.each do |work|
      work_stats(work)
      @work_stats_hash_map[work.id] = {
        progress_annotated: @progress_annotated,
        progress_review: @progress_review,
        progress_completed: @progress_completed
      }
    end

    sort_filtered_data

    if params[:per_page] != '-1'
      @works = @works.paginate(page: params[:page], per_page: params[:per_page] || DEFAULT_WORKS_PER_PAGE)
    end

    @table_export = @collection.works.joins(:table_cells).where.not(table_cells: { work_id: nil }).distinct
  end

  def sort_filtered_data
    case @sorting
    when :page_count
      sorting_arguments = "work_statistics.total_pages #{@ordering}"
    when :indexed_count
      ordered_work_ids = calculate_ordered_work_ids(:progress_annotated)
      ordered_work_ids.reverse! if @ordering == :desc

      sorting_arguments = "FIELD(id, #{ordered_work_ids.join(',')})"
    when :completed_count
      ordered_work_ids = calculate_ordered_work_ids(:progress_completed)
      ordered_work_ids.reverse! if @ordering == :desc

      sorting_arguments = "FIELD(id, #{ordered_work_ids.join(',')})"
    when :reviewed_count
      ordered_work_ids = calculate_ordered_work_ids(:progress_review)
      ordered_work_ids.reverse! if @ordering == :desc

      sorting_arguments = "FIELD(id, #{ordered_work_ids.join(',')})"
    else
      sorting_arguments = "title #{@ordering}"
    end

    @works = @works.reorder(Arel.sql(sorting_arguments))
  end

  def calculate_ordered_work_ids(key)
    @works.sort_by do |work|
      @work_stats_hash_map[work.id][key]
    end.pluck(:id)
  end
end

1	require 'contentdm_translator'	1✔
2
3	class ExportController < ApplicationController	1✔
4	require 'zip'	1✔
5
6	include CollectionHelper	1✔
7	include ExportHelper	1✔
8	include ExportService	1✔
9
10	DEFAULT_WORKS_PER_PAGE = 15	1✔
11
12	before_action :require_owner, only: [:index]	1✔
13
14	def index	1✔
15	filtered_data	21✔
16
17	respond_to do \|format\|	21✔
18	format.html	21✔
19	format.turbo_stream	21✔
20	end
21	end
22
23	def show	1✔
24	xhtml = work_to_xhtml(@work)	4✔
25
26	render text: xhtml, layout: false	4✔
27	end
28
29	def printable	1✔
30	result = Work::Export::Printable.new(	3✔
31	work: @work,
32	format: params[:format],
33	edition: params[:edition],
34	include_metadata: true,
35	include_contributors: true,
36	include_notes: false,
37	preserve_lb: false
38	).call
39
40	if result.success?	3✔
41	send_data(	2✔
42	File.read(result.file),
43	filename: result.filename,
44	content_type: result.content_type
45	)
46
47	cookies['download_finished'] = 'true'	2✔
48	else	1✔
49	head :internal_server_error	1✔
50	end
51	end
52
53	def tei	1✔
54	tei_xml = work_to_tei(@work, current_user)	21✔
55
56	render text: tei_xml, content_type: 'application/xml', layout: false	21✔
57	end
58
59	def subject_details_csv	1✔
60	send_data(	1✔
61	@collection.export_subject_details_as_csv,
62	filename: "fromthepage_subject_details_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
63	type: 'application/csv'
64	)
65	cookies['download_finished'] = 'true'	1✔
66	end
67
68	def subject_coocurrence_csv	1✔
69	send_data(	1✔
70	@collection.export_subject_coocurrence_as_csv,
71	filename: "fromthepage_subject_coocurrence_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
72	type: 'application/csv'
73	)
74	cookies['download_finished'] = 'true'	1✔
75	end
76
77	def subject_distribution_csv	1✔
78	send_data(	1✔
79	@collection.export_subject_distribution_as_csv(@article),
80	filename: "fromthepage_subject_distribution_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
81	type: 'application/csv'
82	)
83	cookies['download_finished'] = 'true'	1✔
84	end
85
86	def subject_index_csv	1✔
87	send_data(	1✔
88	@collection.export_subject_index_as_csv(@collection.works),
89	filename: "fromthepage_subject_index_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
90	type: 'application/csv'
91	)
92	cookies['download_finished'] = 'true'	1✔
93	end
94
95	def work_metadata_csv	1✔
96	filename = params[:filename] ? "#{params[:filename]}.csv" : "fromthepage_work_metadata_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv"	2✔
97	result = Work::Metadata::ExportCsv.new(collection: @collection, works: @collection.works).call	2✔
98
99	send_data(	2✔
100	result.csv_string,
101	filename: filename,
102	type: 'application/csv'
103	)
104	cookies['download_finished'] = 'true'	2✔
105	end
106
107	def table_csv	1✔
108	filename = "fromthepage_tables_export_#{@work.id}_#{Time.now.utc.iso8601}.csv"	3✔
109	collection = @work.collection	3✔
110
111	if collection.field_based?	3✔
112	result = Work::Table::ExportCsv.new(	2✔
113	collection: collection,
114	work_ids: [@work.id]
115	).call
116
117	raise 'Failed to export csv' unless result.success?	2!
118
119	csv_string = result.csv_string	2✔
120	else	1✔
121	csv_string = export_tables_as_csv(@work)	1✔
122	end
123
124	send_data(	3✔
125	csv_string,
126	filename: filename,
127	type: 'text/csv'
128	)
129	cookies['download_finished'] = 'true'	3✔
130	end
131
132	def export_all_tables	1✔
133	filename = "fromthepage_tables_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv"	2✔
134
135	if @collection.field_based?	2✔
136	result = Work::Table::ExportCsv.new(	1✔
137	collection: @collection,
138	work_ids: @collection.works.pluck(:id)
139	).call
140
141	csv_string = result.csv_string	1✔
142	else	1✔
143	csv_string = export_tables_as_csv(@collection)	1✔
144	end
145	send_data(	2✔
146	csv_string,
147	filename: filename,
148	type: 'application/csv'
149	)
150	cookies['download_finished'] = 'true'	2✔
151	end
152
153	def page_plaintext_verbatim	1✔
154	render layout: false, content_type: 'text/plain', plain: @page.verbatim_transcription_plaintext	1✔
155	end
156
157	def page_plaintext_translation_verbatim	1✔
158	render layout: false, content_type: 'text/plain', plain: @page.verbatim_translation_plaintext	1✔
159	end
160
161	def page_plaintext_emended	1✔
162	render layout: false, content_type: 'text/plain', plain: @page.emended_transcription_plaintext	1✔
163	end
164
165	def page_plaintext_translation_emended	1✔
166	render layout: false, content_type: 'text/plain', plain: @page.emended_translation_plaintext	1✔
167	end
168
169	def page_plaintext_searchable	1✔
170	render layout: false, content_type: 'text/plain', plain: @page.search_text	1✔
171	end
172
173	def work_plaintext_verbatim	1✔
174	render layout: false, content_type: 'text/plain', plain: @work.verbatim_transcription_plaintext	3✔
175	end
176
177	def work_plaintext_translation_verbatim	1✔
178	render layout: false, content_type: 'text/plain', plain: @work.verbatim_translation_plaintext	1✔
179	end
180
181	def work_plaintext_emended	1✔
182	render layout: false, content_type: 'text/plain', plain: @work.emended_transcription_plaintext	1✔
183	end
184
185	def work_plaintext_translation_emended	1✔
186	render layout: false, content_type: 'text/plain', plain: @work.emended_translation_plaintext	1✔
187	end
188
189	def work_plaintext_searchable	1✔
190	render layout: false, content_type: 'text/plain', plain: @work.searchable_plaintext	1✔
191	end
192
193	def edit_contentdm_credentials	1✔
194	@sync_target = @collection	1✔
195	@cdm_collection = cdm_collection_for(@sync_target)	1✔
196
197	if ContentdmTranslator.collection_is_cdm?(@sync_target)	1✔
198	begin	×
NEW 199	field_config = ContentdmTranslator.fetch_cdm_field_config(@sync_target)	×
200	@cdm_fulltext_fields = field_config.select { \|e\| e['type'] == 'FTS' }.map { \|e\| [e['name'], e['nick']] }	×
201	@cdm_metadata_fields = field_config.map { \|e\| [e['name'], e['nick']] }	×
202	rescue => e
203	Rails.logger.error("Failed to fetch CONTENTdm field config: #{e.message}")	×
204	end
205	end
206	end
207
208	# TODO: Add specs for this
209	def update_contentdm_credentials	1✔
210	# test credentials
211	license_key = params[:collection][:license_key]	×
212	contentdm_user_name = params[:contentdm_user_name]	×
213	contentdm_password = params[:contentdm_password]	×
214	error_message, _fts_field = ContentdmTranslator.fts_field_for_collection(@collection)	×
215
216	# persist license key and export settings so the user doesn't have to retype them
217	if error_message.blank? \|\| !error_message.match(/license.*invalid/)	×
NEW 218	cdm_collection = cdm_collection_for(@collection)	×
NEW 219	cdm_collection.license_key = license_key	×
NEW 220	cdm_collection.save!	×
221
NEW 222	cdm_setting = cdm_collection.cdm_export_setting \|\| cdm_collection.build_cdm_export_setting	×
223	cdm_setting.transcript_source = params[:transcript_source].presence \|\| CdmExportSetting::HUMAN_ONLY	×
224	cdm_setting.fulltext_field = params[:cdm_fulltext_field].presence	×
225	cdm_setting.include_ai_provenance = params[:include_ai_provenance] == '1'	×
226	cdm_setting.ai_provenance_field = params[:cdm_ai_provenance_field].presence	×
227	cdm_setting.prepend_ai_warning = params[:prepend_ai_warning] == '1'	×
228	cdm_setting.save!	×
229	end
230
231	# redirect to or render edit screen with error
232	if error_message	×
233	flash[:error] = error_message	×
NEW 234	@sync_target = @collection	×
NEW 235	@cdm_collection = cdm_collection_for(@sync_target)	×
NEW 236	render action: :edit_contentdm_credentials, collection_id: @collection.slug	×
UNCOV 237	return	×
238	end
239
240	# pass credentials, FTS field, and search to background job
241	log_file = ContentdmTranslator.log_file(@collection)	×
242	FileUtils.mkdir_p(File.dirname(log_file)) unless Dir.exist? File.dirname(log_file)	×
NEW 243	cmd = "rake fromthepage:cdm_transcript_export[#{@collection.slug}] > #{log_file} 2>&1 &"	×
244	logger.info(cmd)	×
245	system({ 'contentdm_username' => contentdm_user_name, 'contentdm_password' => contentdm_password, 'contentdm_license' => license_key }, cmd)	×
246
247	# display results somehow
248	flash[:notice] = t('.updating_contentdm_message')	×
NEW 249	ajax_redirect_to action: :index, collection_id: @collection.slug	×
250	end
251
252	private	1✔
253
254	def cdm_collection_for(collection_or_set)	1✔
255	collection_or_set.is_a?(DocumentSet) ? collection_or_set.collection : collection_or_set	1!
256	end
257
258	def require_owner	1✔
259	unless user_signed_in? && current_user.like_owner?(@collection)	23✔
260	redirect_to main_app.dashboard_path	2✔
261	end
262	end
263
264	def filtered_data	1✔
265	@sorting = (params[:sort] \|\| 'title').to_sym	21✔
266	@ordering = (params[:order] \|\| 'ASC').downcase.to_sym	21✔
267	@ordering = [:asc, :desc].include?(@ordering) ? @ordering : :desc	21✔
268
269	# Check if there are any translated works in the collection
270	@header = @collection.works.where(supports_translation: true).exists? ? 'Translated' : 'Transcribed'	21✔
271
272	@works = params[:search].blank? ? @collection.works : @collection.search_works(params[:search])	21✔
273	@works = @works.includes(:work_statistic)	21✔
274
275	@work_stats_hash_map = {}	21✔
276	@works.each do \|work\|	21✔
277	work_stats(work)	27✔
278	@work_stats_hash_map[work.id] = {	27✔
279	progress_annotated: @progress_annotated,
280	progress_review: @progress_review,
281	progress_completed: @progress_completed
282	}
283	end
284
285	sort_filtered_data	21✔
286
287	if params[:per_page] != '-1'	21✔
288	@works = @works.paginate(page: params[:page], per_page: params[:per_page] \|\| DEFAULT_WORKS_PER_PAGE)	20✔
289	end
290
291	@table_export = @collection.works.joins(:table_cells).where.not(table_cells: { work_id: nil }).distinct	21✔
292	end
293
294	def sort_filtered_data	1✔
295	case @sorting	21✔
296	when :page_count	1✔
297	sorting_arguments = "work_statistics.total_pages #{@ordering}"	1✔
298	when :indexed_count	2✔
299	ordered_work_ids = calculate_ordered_work_ids(:progress_annotated)	2✔
300	ordered_work_ids.reverse! if @ordering == :desc	2✔
301
302	sorting_arguments = "FIELD(id, #{ordered_work_ids.join(',')})"	2✔
303	when :completed_count	2✔
304	ordered_work_ids = calculate_ordered_work_ids(:progress_completed)	2✔
305	ordered_work_ids.reverse! if @ordering == :desc	2✔
306
307	sorting_arguments = "FIELD(id, #{ordered_work_ids.join(',')})"	2✔
308	when :reviewed_count	2✔
309	ordered_work_ids = calculate_ordered_work_ids(:progress_review)	2✔
310	ordered_work_ids.reverse! if @ordering == :desc	2✔
311
312	sorting_arguments = "FIELD(id, #{ordered_work_ids.join(',')})"	2✔
313	else	14✔
314	sorting_arguments = "title #{@ordering}"	14✔
315	end
316
317	@works = @works.reorder(Arel.sql(sorting_arguments))	21✔
318	end
319
320	def calculate_ordered_work_ids(key)	1✔
321	@works.sort_by do \|work\|	6✔
322	@work_stats_hash_map[work.id][key]	6✔
323	end.pluck(:id)
324	end
325	end

benwbrum / fromthepage / 28600002640

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous