18198125262

Committed 02 Oct 2025 03:43PM UTC coverage: 65.028%. Remained the same

Build # 18198125262

Build Type

push

github

Committed by

web-flow

Commit Message

Rubocop lint remove array whitespaces (#4950)

Coverage Stats

1848 of 3359 branches covered (55.02%)

Branch coverage included in aggregate %.

154 of 201 new or added lines in 62 files covered. (76.62%)

8087 of 11919 relevant lines covered (67.85%)

109.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.44

/app/models/bulk_export.rb

# == Schema Information
#
# Table name: bulk_exports
#
#  id                                 :integer          not null, primary key
#  admin_searches                     :boolean
#  collection_activity                :boolean
#  collection_contributors            :boolean
#  facing_edition_work                :boolean
#  html_page                          :boolean
#  html_work                          :boolean
#  notes_csv                          :boolean
#  organization                       :string(255)      default("by_work")
#  owner_detailed_activity            :boolean
#  owner_mailing_list                 :boolean
#  plaintext_emended_page             :boolean
#  plaintext_emended_work             :boolean
#  plaintext_searchable_page          :boolean
#  plaintext_searchable_work          :boolean
#  plaintext_verbatim_page            :boolean
#  plaintext_verbatim_work            :boolean
#  plaintext_verbatim_zero_index_page :boolean          default(FALSE)
#  report_arguments                   :string(255)
#  static                             :boolean
#  status                             :string(255)
#  subject_csv_collection             :boolean
#  subject_details_csv_collection     :boolean
#  table_csv_collection               :boolean
#  table_csv_work                     :boolean
#  tei_work                           :boolean
#  text_docx_work                     :boolean
#  text_only_pdf_work                 :boolean
#  text_pdf_work                      :boolean
#  use_uploaded_filename              :boolean          default(FALSE)
#  work_metadata_csv                  :boolean          default(FALSE)
#  created_at                         :datetime         not null
#  updated_at                         :datetime         not null
#  collection_id                      :integer
#  document_set_id                    :integer
#  user_id                            :integer          not null
#  work_id                            :integer
#
# Indexes
#
#  index_bulk_exports_on_collection_id    (collection_id)
#  index_bulk_exports_on_document_set_id  (document_set_id)
#  index_bulk_exports_on_user_id          (user_id)
#  index_bulk_exports_on_work_id          (work_id)
#
# Foreign Keys
#
#  fk_rails_...  (collection_id => collections.id)
#  fk_rails_...  (document_set_id => document_sets.id)
#  fk_rails_...  (user_id => users.id)
#  fk_rails_...  (work_id => works.id)
#
class BulkExport < ApplicationRecord
  require 'zip'

  include ExportHelper, ExportService

  store :report_arguments, accessors: [
    :preserve_linebreaks,
    :include_metadata,
    :include_contributors,
    :start_date,
    :end_date,
    :include_notes
  ], coder: JSON

  belongs_to :user
  belongs_to :collection, optional: true
  belongs_to :document_set, optional: true
  belongs_to :work, optional: true

  module Status
    NEW = 'new'
    QUEUED = 'queued'
    PROCESSING = 'processing'
    FINISHED = 'finished'
    CLEANED = 'cleaned'
    ERROR = 'error'
  end

  module Organization
    FORMAT_THEN_WORK = 'by_format'
    WORK_THEN_FORMAT = 'by_work'
  end

  def work_level?
    self.attributes.detect { |k, v| k.match(/_work/) && v==true }
  end

  def page_level?
    self.attributes.detect { |k, v| k.match(/_page/) && v==true }
  end

  def export_to_zip
    self.status = Status::PROCESSING
    self.save

    begin
      if self.work
        works=[self.work]
      elsif self.document_set
        works = self.document_set.works.includes(pages: [:notes, { page_versions: :user }])
      elsif self.collection
        works = Work.includes(pages: [:notes, { page_versions: :user }]).where(collection_id: self.collection.id)
      else
        works = []
      end

      buffer = Zip::OutputStream.open(zip_file_name) do |out|
        write_work_exports(works, out, self.user, self)
        out.close
      end

      self.status = Status::FINISHED
      self.save
    rescue => ex
      self.status = Status::ERROR
      self.save

      raise
    end
  end

  def clean_zip_file
    File.unlink(zip_file_name) if File.exist?(zip_file_name)
    File.unlink(log_file) if File.exist?(log_file)
    self.status = Status::CLEANED
    self.save
  end


  def submit_export_process
    self.status = Status::QUEUED
    self.save
    rake_call = "#{RAKE} fromthepage:process_bulk_export[#{self.id}]  --trace >> #{log_file} 2>&1 &"

    # Nice-up the rake call if settings are present
    rake_call = "nice -n #{NICE_RAKE_LEVEL} stdbuf -oL " << rake_call if NICE_RAKE_ENABLED

    logger.info rake_call
    system('env > /home/fromthepage/environment_logging/env_from_application.log')
    Rails.logger.info("whoami is \n" + `whoami`)
    Rails.logger.info("pwd is \n" + `pwd`)
    Rails.logger.info("ulimit -a is \n" + `/bin/bash -c "ulimit -a"`)
    system(rake_call)
  end

  def log_file
    File.join(zip_file_path, "rake_bulk_export_#{self.id}.log")
  end

  def log_contents
    if File.exist?(log_file)
      File.read(log_file)
    else
      'Log file has been cleaned'
    end
  end

  def zip_file_path
    path = '/tmp/fromthepage_exports'
    FileUtils.mkdir_p(path)

    path
  end

  def zip_file_name
    File.join(zip_file_path, "export_#{self.id}.zip")
  end
end

1	# == Schema Information
2	#
3	# Table name: bulk_exports
4	#
5	# id :integer not null, primary key
6	# admin_searches :boolean
7	# collection_activity :boolean
8	# collection_contributors :boolean
9	# facing_edition_work :boolean
10	# html_page :boolean
11	# html_work :boolean
12	# notes_csv :boolean
13	# organization :string(255) default("by_work")
14	# owner_detailed_activity :boolean
15	# owner_mailing_list :boolean
16	# plaintext_emended_page :boolean
17	# plaintext_emended_work :boolean
18	# plaintext_searchable_page :boolean
19	# plaintext_searchable_work :boolean
20	# plaintext_verbatim_page :boolean
21	# plaintext_verbatim_work :boolean
22	# plaintext_verbatim_zero_index_page :boolean default(FALSE)
23	# report_arguments :string(255)
24	# static :boolean
25	# status :string(255)
26	# subject_csv_collection :boolean
27	# subject_details_csv_collection :boolean
28	# table_csv_collection :boolean
29	# table_csv_work :boolean
30	# tei_work :boolean
31	# text_docx_work :boolean
32	# text_only_pdf_work :boolean
33	# text_pdf_work :boolean
34	# use_uploaded_filename :boolean default(FALSE)
35	# work_metadata_csv :boolean default(FALSE)
36	# created_at :datetime not null
37	# updated_at :datetime not null
38	# collection_id :integer
39	# document_set_id :integer
40	# user_id :integer not null
41	# work_id :integer
42	#
43	# Indexes
44	#
45	# index_bulk_exports_on_collection_id (collection_id)
46	# index_bulk_exports_on_document_set_id (document_set_id)
47	# index_bulk_exports_on_user_id (user_id)
48	# index_bulk_exports_on_work_id (work_id)
49	#
50	# Foreign Keys
51	#
52	# fk_rails_... (collection_id => collections.id)
53	# fk_rails_... (document_set_id => document_sets.id)
54	# fk_rails_... (user_id => users.id)
55	# fk_rails_... (work_id => works.id)
56	#
57	class BulkExport < ApplicationRecord	1✔
58	require 'zip'	1✔
59
60	include ExportHelper, ExportService	1✔
61
62	store :report_arguments, accessors: [	1✔
63	:preserve_linebreaks,
64	:include_metadata,
65	:include_contributors,
66	:start_date,
67	:end_date,
68	:include_notes
69	], coder: JSON
70
71	belongs_to :user	1✔
72	belongs_to :collection, optional: true	1✔
73	belongs_to :document_set, optional: true	1✔
74	belongs_to :work, optional: true	1✔
75
76	module Status	1✔
77	NEW = 'new'	1✔
78	QUEUED = 'queued'	1✔
79	PROCESSING = 'processing'	1✔
80	FINISHED = 'finished'	1✔
81	CLEANED = 'cleaned'	1✔
82	ERROR = 'error'	1✔
83	end
84
85	module Organization	1✔
86	FORMAT_THEN_WORK = 'by_format'	1✔
87	WORK_THEN_FORMAT = 'by_work'	1✔
88	end
89
90	def work_level?	1✔
91	self.attributes.detect { \|k, v\| k.match(/_work/) && v==true }	×
92	end
93
94	def page_level?	1✔
95	self.attributes.detect { \|k, v\| k.match(/_page/) && v==true }	×
96	end
97
98	def export_to_zip	1✔
99	self.status = Status::PROCESSING	×
100	self.save	×
101
102	begin
103	if self.work	×
NEW 104	works=[self.work]	×
105	elsif self.document_set	×
NEW 106	works = self.document_set.works.includes(pages: [:notes, { page_versions: :user }])	×
107	elsif self.collection	×
NEW 108	works = Work.includes(pages: [:notes, { page_versions: :user }]).where(collection_id: self.collection.id)	×
109	else	×
110	works = []	×
111	end
112
113	buffer = Zip::OutputStream.open(zip_file_name) do \|out\|	×
114	write_work_exports(works, out, self.user, self)	×
115	out.close	×
116	end
117
118	self.status = Status::FINISHED	×
119	self.save	×
120	rescue => ex
121	self.status = Status::ERROR	×
122	self.save	×
123
124	raise	×
125	end
126	end
127
128	def clean_zip_file	1✔
129	File.unlink(zip_file_name) if File.exist?(zip_file_name)	×
130	File.unlink(log_file) if File.exist?(log_file)	×
131	self.status = Status::CLEANED	×
132	self.save	×
133	end
134
135
136	def submit_export_process	1✔
137	self.status = Status::QUEUED	2✔
138	self.save	2✔
139	rake_call = "#{RAKE} fromthepage:process_bulk_export[#{self.id}] --trace >> #{log_file} 2>&1 &"	2✔
140
141	# Nice-up the rake call if settings are present
142	rake_call = "nice -n #{NICE_RAKE_LEVEL} stdbuf -oL " << rake_call if NICE_RAKE_ENABLED	2!
143
144	logger.info rake_call	2✔
145	system('env > /home/fromthepage/environment_logging/env_from_application.log')	2✔
146	Rails.logger.info("whoami is \n" + `whoami`)	2✔
147	Rails.logger.info("pwd is \n" + `pwd`)	2✔
148	Rails.logger.info("ulimit -a is \n" + `/bin/bash -c "ulimit -a"`)	2✔
149	system(rake_call)	2✔
150	end
151
152	def log_file	1✔
153	File.join(zip_file_path, "rake_bulk_export_#{self.id}.log")	2✔
154	end
155
156	def log_contents	1✔
157	if File.exist?(log_file)	×
158	File.read(log_file)	×
159	else	×
160	'Log file has been cleaned'	×
161	end
162	end
163
164	def zip_file_path	1✔
165	path = '/tmp/fromthepage_exports'	6✔
166	FileUtils.mkdir_p(path)	6✔
167
168	path	6✔
169	end
170
171	def zip_file_name	1✔
172	File.join(zip_file_path, "export_#{self.id}.zip")	3✔
173	end
174	end

benwbrum / fromthepage / 18198125262

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous