15930835603

Committed 27 Jun 2025 04:11PM UTC coverage: 60.826% (-0.05%) from 60.876%

Build # 15930835603

Build Type

push

github

Committed by

web-flow

Commit Message

4702 - ContentDM push retry (#4724)

* 4702 - ContentDM push retry

* 4702 - Increase back-off times

Coverage Stats

1624 of 3268 branches covered (49.69%)

Branch coverage included in aggregate %.

1 of 13 new or added lines in 1 file covered. (7.69%)

7467 of 11678 relevant lines covered (63.94%)

78.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.87

/lib/contentdm_translator.rb

module ContentdmTranslator

  def self.update_work_from_cdm(work, ocr_correction=false)
    # find the work manifest -- bail out if there is none
    return unless work.sc_manifest
    # make sure the manifest is cdm
    return unless iiif_manifest_is_cdm? work.sc_manifest.at_id

    if ocr_correction
      # get the fts field for this collection
      fts_error, fts_field = fts_field_for_collection(work.collection)
      if fts_error
        puts "Error retrieving Full-Text Search field: #{fts_error}\n"
      end
    end
    # for each page
    work.pages.each do |page|
      update_page_from_cdm(page, ocr_correction, fts_field)
    end
    work.ocr_correction=ocr_correction
    work.save!
  end

  def self.update_page_from_cdm(page, ocr_correction, fts_field)
    # fetch the cdm metadata
    info = fetch_cdm_info(page)
    # prune the boilerplate
    metadata = metadata_from_cdm_info(info)
    # store the metadata on the page
    page_columns = { metadata: metadata }

    if ocr_correction
      ocr = ocr_from_cdm_info(info, fts_field)
      page_columns[:source_text] = ocr.encode(:xml => :text) if ocr
    end

    page.update_columns(page_columns)
  end

  def self.fetch_cdm_info(page)
    cdm_url = page_at_id_to_cdm_item_info(page.sc_canvas.sc_canvas_id)
    cdm_response = URI.open(cdm_url).read
    JSON.parse(cdm_response)
  end

  def self.fetch_cdm_field_config(collection)
    cdm_url = collection_to_cdm_field_config(collection)
    cdm_response = URI.open(cdm_url).read
    JSON.parse(cdm_response)
  end

  ITEM_INFO_DENYLIST = [
    "descri",
    "date",
    "creato",
    "subjec",
    "relate",
    "type",
    "publis",
    "langua",
    "rights",
    "transc",
    "contac",
    "fullrs",
    "find",
    "dmaccess",
    "dmimage",
    "dmcreated",
    "dmmodified",
    "dmoclcno",
    "restrictionCode",
    "cdmfilesize",
    "cdmfilesizeformatted",
    "cdmprintpdf",
    "cdmhasocr",
    "cdmisnewspaper"]

  def self.metadata_from_cdm_info(info)
    # only return useful and unique things
    info.except(*ITEM_INFO_DENYLIST)
  end

  def self.ocr_from_cdm_info(info, fts_field)
    transcript = info[fts_field]
    if transcript.kind_of? String
      transcript
    else
      nil
    end
  end

  def self.page_at_id_to_cdm_item_info(at_id)
    cdm = at_id.sub(/cdm/, 'server')
    cdm.sub!(/(digital\/)?iiif/, 'dmwebservices/index.php?q=dmGetItemInfo')
    cdm.sub!(/\/canvas\/c\d*/, '/json')
    cdm.sub!(/:(\d+)/, '/\1') # handle coollection:id format instead of old collection/id

    cdm
  end

  def self.collection_to_cdm_field_config(collection)
    at_id = collection.pages.joins(:sc_canvas).reorder('pages.created_on').last.sc_canvas.sc_canvas_id
    cdm = at_id.sub(/cdm/, 'server')
    cdm.sub!(/(digital\/)?iiif/, 'dmwebservices/index.php?q=dmGetCollectionFieldInfo')
    cdm.sub!(/(:\d+)?\/canvas\/c\d*/, '/json')

    cdm
  end



  def self.iiif_manifest_is_cdm?(at_id)
    at_id.match(/contentdm.oclc.org/) || at_id.match(/iiif\/info\/\w+\/\d+\/manifest.json/)
  end

  def self.cdm_item_info_from_iiif(at_id)
    cdm = at_id.sub(/cdm/, 'server')
    cdm.sub!(/digital\/iiif-info/, 'dmwebservices/index.php?q=dmGetItemInfo')
  end

  def self.collection_is_cdm?(collection)
    imported_work = collection.works.joins(:sc_manifest).last
    imported_work && iiif_manifest_is_cdm?(imported_work.sc_manifest.at_id)
  end

  def self.fts_field_for_collection(collection)
    field_config = fetch_cdm_field_config(collection)
    fts_field = field_config.detect { |element| element["type"] == "FTS"}
    if fts_field
      fts = fts_field['nick']
      error = nil
    else
      fts = nil
      error = "No full-text search (FTS) fields were configured on CONTENTdm collection!"
    end
    return error, fts
  end

  def self.export_work_to_cdm_with_retry(work, username, password, license)
    max_delay = 21_600
    delay = 300

    begin
      ContentdmTranslator.export_work_to_cdm(work, username, password, license)
    rescue Net::ReadTimeout => e
      delay_to_use = [delay, max_delay].min
      print "Net::ReadTimeout: Retrying in #{delay_to_use} seconds... (#{e.message})"

      sleep(delay_to_use)

      delay = (delay * 1.5).round
      if delay > max_delay
        print "Net::ReadTimeout: Max retry delay reached, giving up. (#{e.message})"
      else
        retry
      end
    end
  end

  def self.export_work_to_cdm(work, username, password, license)
    error, fieldname = fts_field_for_collection(work.collection)
    if error
      puts "Error retrieving Full-Text Search field: #{error}\n"
      exit
    end

    soap_client = Savon.client(:log=>true, filters: [:password], :wsdl => 'https://worldcat.org/webservices/contentdm/catcher?wsdl', follow_redirects: true)
    work.pages.each do |page|
      canvas_at_id = page.sc_canvas.sc_canvas_id
      manifest_at_id = work.sc_manifest.at_id
      puts "\nUpdating #{cdm_collection(manifest_at_id)}\trecord #{cdm_record(canvas_at_id)}\tfrom #{page.title}\t#{page.id}\t#{work.title} at #{Time.current.strftime('%Y-%m-%d %I:%M %p')}.  CONTENTdm response:"
      metadata_wrapper = {
        'metadataList' => {
          'metadata' => [
            { :field => 'dmrecord', :value => cdm_record(canvas_at_id)},
            { :field => fieldname, :value => page.verbatim_transcription_plaintext}
          ]
        }
      }

      message = {
        :cdmurl => "http://#{cdm_server(manifest_at_id)}:8888",
        :username => username,
        :password => password,
        :license => license,
        :collection => cdm_collection(manifest_at_id),
        :metadata => metadata_wrapper,
        :action => 'edit'
      }
      resp = soap_client.call(:process_conten_tdm, :message => message )

      puts resp.to_hash[:process_conten_tdm_response][:return]

    end
  end

  def self.log_file(collection)
    File.join(Rails.root, 'public', 'imports', "cdm_sync_#{collection.id}.log")
  end

  def self.log_contents(collection)
    STDOUT.flush
    File.read(log_file(collection))
  end

  private

  def self.cdm_server(at_id)
    at_id.sub(/https:\/\/cdm/,'server').sub(/\/.*/,'')
  end

  def self.cdm_collection(at_id)
    if at_id.match(/.*iiif\/info\//)
      at_id.sub(/.*iiif\/info\//, '').sub(/\/\d+\/manifest.json/, '')
    elsif at_id.match(/.*iiif\/2\//)
      at_id.sub(/.*iiif\/2\//, '').sub(/:.*/,'')
    else # match https://cdm17168.contentdm.oclc.org/iiif/WFP:997/manifest.json
      at_id.sub(/.*iiif\//, '').sub(/:.*/,'')
    end
  end

  def self.cdm_record(at_id)
    at_id.sub(/\/canvas\/.*/,'').sub(/^.*\//, '').sub(/^.*:/, '')
  end

  def self.get_cdm_host_from_url(host)
    matches = host.match(/https?:\/\/(cdm\d+)/)
    return matches[1] if matches

    res = URI.open("#{host}/iiif/info/manifest.json").read
    res_json = JSON.parse(res)
    url = res_json['@id'] || nil

    if url
      matches = url.match(/https?:\/\/(cdm\d+)/)
      return matches[1] if matches
    end
    nil
  end

  def self.cdm_url_to_iiif(url)
    uri = URI(url)

    server = get_cdm_host_from_url("#{uri.scheme}://#{uri.host}")
    raise "ContentDM URLs must be of the form http://cdmNNNNN.contentdm.oclc.org/..." if server.nil?

    matches = uri.path.match(/.*collection\/(\w+)(?:\/id\/(\d+))?/)

    if matches
      collection = matches[1]
      record = matches[2]
    end

    # support back-level CONTENTdm IIIF presentation implementation
    if server && collection && record
      new_uri = "https://#{server}.contentdm.oclc.org/iiif/info/#{collection}/#{record}/manifest.json"
    elsif server && collection
      new_uri = "https://#{server}.contentdm.oclc.org/iiif/info/#{collection}/manifest.json"
    elsif server
      new_uri = "https://#{server}.contentdm.oclc.org/iiif/info/manifest.json"
    else
      raise "ContentDM URLs must be of the form http://cdmNNNNN.contentdm.oclc.org/..."
    end

    begin
      URI.open(new_uri)
    rescue OpenURI::HTTPError
      if server && collection && record
        # https://cdm17217.contentdm.oclc.org/iiif/2/voter1867:4764/manifest.json
        new_uri = "https://#{server}.contentdm.oclc.org/iiif/2/#{collection}:#{record}/manifest.json"
      elsif server && collection
        # https://cdm17217.contentdm.oclc.org/iiif/2/voter1867/manifest.json
        new_uri = "https://#{server}.contentdm.oclc.org/iiif/2/#{collection}/manifest.json"
      else
        # https://cdm17217.contentdm.oclc.org/iiif/2/manifest.json
        new_uri = "https://#{server}.contentdm.oclc.org/iiif/2/manifest.json"
      end

    end

    new_uri
  end

  def self.sample_manifest(collection)
    imported_work = collection.works.joins(:sc_manifest).last

    imported_work && imported_work.sc_manifest
  end
end

1	module ContentdmTranslator	1✔
2
3	def self.update_work_from_cdm(work, ocr_correction=false)	1✔
4	# find the work manifest -- bail out if there is none
5	return unless work.sc_manifest	×
6	# make sure the manifest is cdm
7	return unless iiif_manifest_is_cdm? work.sc_manifest.at_id	×
8
9	if ocr_correction	×
10	# get the fts field for this collection	×
11	fts_error, fts_field = fts_field_for_collection(work.collection)	×
12	if fts_error	×
13	puts "Error retrieving Full-Text Search field: #{fts_error}\n"	×
14	end
15	end
16	# for each page
17	work.pages.each do \|page\|	×
18	update_page_from_cdm(page, ocr_correction, fts_field)	×
19	end
20	work.ocr_correction=ocr_correction	×
21	work.save!	×
22	end
23
24	def self.update_page_from_cdm(page, ocr_correction, fts_field)	1✔
25	# fetch the cdm metadata
26	info = fetch_cdm_info(page)	×
27	# prune the boilerplate
28	metadata = metadata_from_cdm_info(info)	×
29	# store the metadata on the page
30	page_columns = { metadata: metadata }	×
31
32	if ocr_correction	×
33	ocr = ocr_from_cdm_info(info, fts_field)	×
34	page_columns[:source_text] = ocr.encode(:xml => :text) if ocr	×
35	end
36
37	page.update_columns(page_columns)	×
38	end
39
40	def self.fetch_cdm_info(page)	1✔
41	cdm_url = page_at_id_to_cdm_item_info(page.sc_canvas.sc_canvas_id)	×
42	cdm_response = URI.open(cdm_url).read	×
43	JSON.parse(cdm_response)	×
44	end
45
46	def self.fetch_cdm_field_config(collection)	1✔
47	cdm_url = collection_to_cdm_field_config(collection)	×
48	cdm_response = URI.open(cdm_url).read	×
49	JSON.parse(cdm_response)	×
50	end
51
52	ITEM_INFO_DENYLIST = [
53	"descri",	1✔
54	"date",
55	"creato",
56	"subjec",
57	"relate",
58	"type",
59	"publis",
60	"langua",
61	"rights",
62	"transc",
63	"contac",
64	"fullrs",
65	"find",
66	"dmaccess",
67	"dmimage",
68	"dmcreated",
69	"dmmodified",
70	"dmoclcno",
71	"restrictionCode",
72	"cdmfilesize",
73	"cdmfilesizeformatted",
74	"cdmprintpdf",
75	"cdmhasocr",
76	"cdmisnewspaper"]
77
78	def self.metadata_from_cdm_info(info)	1✔
79	# only return useful and unique things
80	info.except(*ITEM_INFO_DENYLIST)	×
81	end
82
83	def self.ocr_from_cdm_info(info, fts_field)	1✔
84	transcript = info[fts_field]	×
85	if transcript.kind_of? String	×
86	transcript	×
87	else
88	nil
89	end
90	end
91
92	def self.page_at_id_to_cdm_item_info(at_id)	1✔
93	cdm = at_id.sub(/cdm/, 'server')	×
94	cdm.sub!(/(digital\/)?iiif/, 'dmwebservices/index.php?q=dmGetItemInfo')	×
95	cdm.sub!(/\/canvas\/c\d*/, '/json')	×
96	cdm.sub!(/:(\d+)/, '/\1') # handle coollection:id format instead of old collection/id	×
97
98	cdm	×
99	end
100
101	def self.collection_to_cdm_field_config(collection)	1✔
102	at_id = collection.pages.joins(:sc_canvas).reorder('pages.created_on').last.sc_canvas.sc_canvas_id	×
103	cdm = at_id.sub(/cdm/, 'server')	×
104	cdm.sub!(/(digital\/)?iiif/, 'dmwebservices/index.php?q=dmGetCollectionFieldInfo')	×
105	cdm.sub!(/(:\d+)?\/canvas\/c\d*/, '/json')	×
106
107	cdm	×
108	end
109
110
111
112	def self.iiif_manifest_is_cdm?(at_id)	1✔
113	at_id.match(/contentdm.oclc.org/) \|\| at_id.match(/iiif\/info\/\w+\/\d+\/manifest.json/)	11✔
114	end
115
116	def self.cdm_item_info_from_iiif(at_id)	1✔
117	cdm = at_id.sub(/cdm/, 'server')	×
118	cdm.sub!(/digital\/iiif-info/, 'dmwebservices/index.php?q=dmGetItemInfo')	×
119	end
120
121	def self.collection_is_cdm?(collection)	1✔
122	imported_work = collection.works.joins(:sc_manifest).last	9✔
123	imported_work && iiif_manifest_is_cdm?(imported_work.sc_manifest.at_id)	9✔
124	end
125
126	def self.fts_field_for_collection(collection)	1✔
127	field_config = fetch_cdm_field_config(collection)	×
128	fts_field = field_config.detect { \|element\| element["type"] == "FTS"}	×
129	if fts_field	×
130	fts = fts_field['nick']	×
131	error = nil	×
132	else	×
133	fts = nil	×
134	error = "No full-text search (FTS) fields were configured on CONTENTdm collection!"	×
135	end
136	return error, fts	×
137	end
138
139	def self.export_work_to_cdm_with_retry(work, username, password, license)	1✔
NEW 140	max_delay = 21_600	×
NEW 141	delay = 300	×
142
143	begin
NEW 144	ContentdmTranslator.export_work_to_cdm(work, username, password, license)	×
NEW 145	rescue Net::ReadTimeout => e	×
NEW 146	delay_to_use = [delay, max_delay].min	×
NEW 147	print "Net::ReadTimeout: Retrying in #{delay_to_use} seconds... (#{e.message})"	×
148
NEW 149	sleep(delay_to_use)	×
150
NEW 151	delay = (delay * 1.5).round	×
NEW 152	if delay > max_delay	×
NEW 153	print "Net::ReadTimeout: Max retry delay reached, giving up. (#{e.message})"	×
154	else	×
NEW 155	retry	×
156	end
157	end
158	end
159
160	def self.export_work_to_cdm(work, username, password, license)	1✔
161	error, fieldname = fts_field_for_collection(work.collection)	×
162	if error	×
163	puts "Error retrieving Full-Text Search field: #{error}\n"	×
164	exit	×
165	end
166
167	soap_client = Savon.client(:log=>true, filters: [:password], :wsdl => 'https://worldcat.org/webservices/contentdm/catcher?wsdl', follow_redirects: true)	×
168	work.pages.each do \|page\|	×
169	canvas_at_id = page.sc_canvas.sc_canvas_id	×
170	manifest_at_id = work.sc_manifest.at_id	×
NEW 171	puts "\nUpdating #{cdm_collection(manifest_at_id)}\trecord #{cdm_record(canvas_at_id)}\tfrom #{page.title}\t#{page.id}\t#{work.title} at #{Time.current.strftime('%Y-%m-%d %I:%M %p')}. CONTENTdm response:"	×
172	metadata_wrapper = {
173	'metadataList' => {	×
174	'metadata' => [
175	{ :field => 'dmrecord', :value => cdm_record(canvas_at_id)},
176	{ :field => fieldname, :value => page.verbatim_transcription_plaintext}
177	]
178	}
179	}
180
181	message = {
182	:cdmurl => "http://#{cdm_server(manifest_at_id)}:8888",	×
183	:username => username,
184	:password => password,
185	:license => license,
186	:collection => cdm_collection(manifest_at_id),
187	:metadata => metadata_wrapper,
188	:action => 'edit'
189	}
190	resp = soap_client.call(:process_conten_tdm, :message => message )	×
191
192	puts resp.to_hash[:process_conten_tdm_response][:return]	×
193
194	end
195	end
196
197	def self.log_file(collection)	1✔
198	File.join(Rails.root, 'public', 'imports', "cdm_sync_#{collection.id}.log")	×
199	end
200
201	def self.log_contents(collection)	1✔
202	STDOUT.flush	×
203	File.read(log_file(collection))	×
204	end
205
206	private	1✔
207
208	def self.cdm_server(at_id)	1✔
209	at_id.sub(/https:\/\/cdm/,'server').sub(/\/.*/,'')	×
210	end
211
212	def self.cdm_collection(at_id)	1✔
213	if at_id.match(/.*iiif\/info\//)	×
214	at_id.sub(/.*iiif\/info\//, '').sub(/\/\d+\/manifest.json/, '')	×
215	elsif at_id.match(/.*iiif\/2\//)	×
216	at_id.sub(/.iiif\/2\//, '').sub(/:./,'')	×
217	else # match https://cdm17168.contentdm.oclc.org/iiif/WFP:997/manifest.json	×
218	at_id.sub(/.iiif\//, '').sub(/:./,'')	×
219	end
220	end
221
222	def self.cdm_record(at_id)	1✔
223	at_id.sub(/\/canvas\/./,'').sub(/^.\//, '').sub(/^.*:/, '')	×
224	end
225
226	def self.get_cdm_host_from_url(host)	1✔
227	matches = host.match(/https?:\/\/(cdm\d+)/)	12✔
228	return matches[1] if matches	12✔
229
230	res = URI.open("#{host}/iiif/info/manifest.json").read	9✔
231	res_json = JSON.parse(res)	8✔
232	url = res_json['@id'] \|\| nil	8✔
233
234	if url	8!
235	matches = url.match(/https?:\/\/(cdm\d+)/)	8✔
236	return matches[1] if matches	8!
237	end
238	nil
239	end
240
241	def self.cdm_url_to_iiif(url)	1✔
242	uri = URI(url)	12✔
243
244	server = get_cdm_host_from_url("#{uri.scheme}://#{uri.host}")	12✔
245	raise "ContentDM URLs must be of the form http://cdmNNNNN.contentdm.oclc.org/..." if server.nil?	11!
246
247	matches = uri.path.match(/.*collection\/(\w+)(?:\/id\/(\d+))?/)	11✔
248
249	if matches	11✔
250	collection = matches[1]	8✔
251	record = matches[2]	8✔
252	end
253
254	# support back-level CONTENTdm IIIF presentation implementation
255	if server && collection && record	11✔
256	new_uri = "https://#{server}.contentdm.oclc.org/iiif/info/#{collection}/#{record}/manifest.json"	4✔
257	elsif server && collection	7✔
258	new_uri = "https://#{server}.contentdm.oclc.org/iiif/info/#{collection}/manifest.json"	4✔
259	elsif server	3✔
260	new_uri = "https://#{server}.contentdm.oclc.org/iiif/info/manifest.json"	3✔
261	else	×
262	raise "ContentDM URLs must be of the form http://cdmNNNNN.contentdm.oclc.org/..."	×
263	end
264
265	begin
266	URI.open(new_uri)	11✔
267	rescue OpenURI::HTTPError
268	if server && collection && record	1✔
269	# https://cdm17217.contentdm.oclc.org/iiif/2/voter1867:4764/manifest.json	1✔
270	new_uri = "https://#{server}.contentdm.oclc.org/iiif/2/#{collection}:#{record}/manifest.json"	1!
271	elsif server && collection	×
272	# https://cdm17217.contentdm.oclc.org/iiif/2/voter1867/manifest.json	×
273	new_uri = "https://#{server}.contentdm.oclc.org/iiif/2/#{collection}/manifest.json"	×
274	else
275	# https://cdm17217.contentdm.oclc.org/iiif/2/manifest.json	×
276	new_uri = "https://#{server}.contentdm.oclc.org/iiif/2/manifest.json"	×
277	end
278
279	end
280
281	new_uri	11✔
282	end
283
284	def self.sample_manifest(collection)	1✔
285	imported_work = collection.works.joins(:sc_manifest).last	×
286
287	imported_work && imported_work.sc_manifest	×
288	end
289	end

benwbrum / fromthepage / 15930835603

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous