6993595b-7e17-4f96-a203-b3db1eaf8e8a

Committed 28 Feb 2025 10:32PM UTC coverage: 83.912% (+0.6%) from 83.356%

Build # 6993595b-7e17-4f96-a203-b3db1eaf8e8a

Build Type

push

circleci

Committed by

christinach

Commit Message

add additional test scenario for pair_880_errors

Run Details

1862 of 2219 relevant lines covered (83.91%)

3.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.31

/lib/marc_cleanup/variable_fields.rb

# frozen_string_literal: true

module MarcCleanup
  ### Remove non-numerical strings and append a new 020$q with the string
  def new_020_q(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn_parts = /^\s*([\d\-]+)\s*(\(.*?\))\s*$/.match(subfield.value)
        next if isbn_parts.nil?
        subfield.value = isbn_parts[1]
        f020.append(MARC::Subfield.new('q', isbn_parts[2]))
      end
    end
    record
  end

  ### Convert ISBN-10 to ISBN-13
  def isbn10_to_13(isbn)
    stem = isbn[0..8]
    return nil if stem =~ /\D/

    existing_check = isbn[9]
    return nil if existing_check && existing_check != checkdigit_10(stem)

    main = ISBN13PREFIX + stem
    checkdigit = checkdigit_13(main)
    main + checkdigit
  end

  ### Calculate check digit for ISBN-10
  def checkdigit_10(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += digit.to_i * (10 - int_index)
      int_index += 1
    end
    mod = (11 - (int_sum % 11)) % 11
    mod == 10 ? 'X' : mod.to_s
  end

  ### Calculate check digit for ISBN-13
  def checkdigit_13(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    ((10 - (int_sum % 10)) % 10).to_s
  end

  ### Normalize ISBN-13
  def isbn13_normalize(raw_isbn)
    int_sum = 0
    stem = raw_isbn[0..11]
    return nil if stem =~ /\D/

    int_index = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    checkdigit = checkdigit_13(stem)
    return nil if raw_isbn[12] && raw_isbn[12] != checkdigit

    stem + checkdigit
  end

  ### Normalize any given string that is supposed to include an ISBN
  def isbn_normalize(isbn)
    return nil unless isbn

    raw_isbn = isbn.dup
    raw_isbn.delete!('-')
    raw_isbn.delete!('\\')
    raw_isbn.gsub!(/\([^\)]*\)/, '')
    raw_isbn.gsub!(/^(.*)\$c.*$/, '\1')
    raw_isbn.gsub!(/^(.*)\$q.*$/, '\1')
    raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')
    if raw_isbn =~ /^978/
      raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')
      raw_isbn.delete!(' ')
    else
      raw_isbn.gsub!(/([0-9])\s*([0-9]{4})\s*([0-9]{4})\s*([0-9xX]).*$/, '\1\2\3\4')
    end
    raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')
    raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')
    if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/
      raw_isbn = raw_isbn.ljust(9, '0')
    end
    valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits
    return nil unless valid_lengths.include? raw_isbn.length

    if raw_isbn.length < 12
      isbn10_to_13(raw_isbn)
    else
      isbn13_normalize(raw_isbn)
    end
  end

  ### If the ISBN is invalid, change the subfield code to z
  ### Otherwise, replace ISBN with normalized ISBN
  def move_invalid_isbn(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn = subfield.value
        normalized_isbn = isbn_normalize(isbn)
        if normalized_isbn
          subfield.value = normalized_isbn
        else
          subfield.code = 'z'
        end
      end
    end
    record
  end

  # check the 041 field for errors
  # 041 is a language code
  def f041_errors?(record)
    f041 = record.fields('041')
    return false if f041.empty?

    f041.each do |field|
      field.subfields.each do |subfield|
        val = subfield.value
        return true if (val.size > 3) && (val.size % 3).zero?
      end
    end
    false
  end

  # http://www.loc.gov/standards/valuelist/marcauthen.html
  def auth_codes_f042
    %w[
      anuc croatica dc dhca dlr
      gamma gils gnd1 gnd2 gnd3 gnd4 gnd5 gnd6 gnd7 gndz isds/c issnuk
      lacderived lc lcac lccopycat lccopycat-nm lcd lcderive
      lchlas lcllh lcnccp lcnitrate lcnuc lcode
      msc natgaz nbr nlc nlmcopyc norbibl nsdp nst ntccf nznb
      pcc premarc reveal sanb scipio toknb
      ukblcatcopy ukblderived ukblproject ukblsr ukscp
      xisds/c xissnuk xlc xnlc xnsdp
    ]
  end

  def auth_code_error?(record)
    return false unless record['042']
    return true if record.fields('042').size > 1

    record['042'].subfields.each do |subfield|
      next if subfield.code != 'a'
      return true unless auth_codes_f042.include?(subfield.value)
    end
    false
  end

  def empty_subfields?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      field.subfields.each do |subfield|
        return true if subfield.value =~ /^[[:blank:]]*$/
      end
    end
    false
  end

  def extra_spaces?(record)
    blank_regex = /^.*[[:blank:]]{2,}.*$|^.*[[:blank:]]+$|^[[:blank:]]+(.*)$/
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag != '010'

      case field.tag
      when /[1-469]..|0[2-9].|01[1-9]|7[0-5].|5[0-24-9].|53[0-24-9]/
        field.subfields.each do |subfield|
          return true if subfield.value =~ blank_regex
        end
      when '533'
        field.subfields.each do |subfield|
          next if subfield.code == '7'

          return true if subfield.value =~ blank_regex
        end
      when /7[6-8]./
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[a-v3-8]/

          return true if subfield.value =~ blank_regex
        end
      when /8../
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[^w7]/

          return true if subfield.value =~ blank_regex
        end
      end
    end
    false
  end

  def extra_space_gsub(string)
    string.gsub!(/([[:blank:]]){2,}/, '\1')
    string.gsub!(/^(.*)[[:blank:]]+$/, '\1')
    string.gsub(/^[[:blank:]]+(.*)$/, '\1')
  end

  ### Remove extra spaces from all fields that are not positionally defined
  def extra_space_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag != '010'

      case field.tag
      when /^[1-469]..|0[2-9].|01[1-9]|7[0-5].|5[0-24-9].|53[0-24-9]/
        field.subfields.each do |subfield|
          next if subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      when '533'
        field.subfields.each do |subfield|
          next if subfield.code == '7' || subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      when /^7[6-8]./
        field.subfields.each do |subfield|
          next if subfield.code =~ /[^a-v3-8]/ || subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      when /^8../
        field.subfields.each do |subfield|
          next if %w[w 7].include?(subfield.code) || subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      end
    end
    record
  end

  def multiple_no_040?(record)
    record.fields('040').size != 1
  end

  def multiple_no_040b?(record)
    f040 = record.fields('040')
    return true if f040.size != 1

    f040 = f040.first
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return true if b040.size != 1

    b040.first.value.match?(/^\s*$/)
  end

  def f046_errors?(record)
    subf_codes = %w[b c d e]
    subf_a_values = %w[r s p t x q n i k r m t x n]
    f046 = record.fields('046')
    return false if f046.empty?

    f046.each do |field|
      codes = field.subfields.map(&:code)
      return true if field['a'] && !subf_a_values.include?(field['a'])
      return true if field['a'].nil? && (subf_codes & codes).size.positive?
    end
    false
  end

  def multiple_no_245?(record)
    record.fields('245').size != 1
  end

  def missing_040c?(record)
    return true unless record['040'] && record['040']['c']

    false
  end

  def pair_880_errors?(record)
    return true if record.fields('880').select { |field| field['6'].nil? }.size.positive?

    pair_880s = f880_pairings(record)
    linked_fields = linked_field_pairings(record)
    pair_880s.uniq != pair_880s || pair_880s.uniq.sort != linked_fields.uniq.sort
  end

  def f880_pairings(record)
    target_fields = record.fields('880').select do |field|
      field['6'] =~ /^[0-9]{3}-[0-9][1-9]/
    end
    target_fields.map do |field|
      field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')
    end
  end

  def linked_field_pairings(record)
    target_fields = record.fields('010'..'899').select do |field|
      field.tag != '880' && field['6']
    end
    target_fields.map do |field|
      "#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1')}"
    end
  end

  def has_130_240?(record)
    (%w[130 240] - record.tags).empty?
  end

  def multiple_1xx?(record)
    record.fields('100'..'199').size > 1
  end

  def relator_chars?(record)
    record.fields.each do |field|
      if field.tag =~ /[17][01]0/
        field.subfields.each do |subfield|
          next unless subfield.code == 'e' && subfield.value =~ /.*[^a-z, \.].*/

          return true
        end
      elsif field.tag =~ /[17]11/
        field.subfields.each do |subfield|
          next unless subfield.code == 'j' && subfield.value =~ /.*[^a-z, \.].*/

          return true
        end
      end
    end
    false
  end

  def x00_subfq?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      field.subfields.each do |subfield|
        next unless subfield.code == 'q' && subfield.value =~ /^(?!\([^\)]*\))$/

        return true
      end
    end
    false
  end

  def no_comma_x00?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      subfx_d_index = code_array.index(/.d/)
      return true if subfx_d_index && field.subfields[subfx_d_index].value =~ /^.*[^,]$/
    end
    false
  end

  def relator_comma?(record)
    comma_regex = /^.*[^,]$/
    record.fields.each do |field|
      next unless field.tag =~ /[17][01][01]/

      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      if field.tag =~ /[17][01]0/
        subfx_e_index = code_array.index(/.e/)
        return true if subfx_e_index && field.subfields[subfx_e_index].value =~ comma_regex
      elsif field.tag =~ /[17]11/
        subfx_j_index = code_array.index(/.j/)
        return true if subfx_j_index && field.subfields[subfx_j_index].value =~ comma_regex
      end
    end
    false
  end

  def heading_end_punct?(record)
    punct_regex = /.*[^"\).\!\?\-]$/
    record.fields.each do |field|
      next unless field.tag =~ /^[167][0-5].$/ && field.indicator2 =~ /[^47]/

      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      last_heading_subfield_index = code_array.index(/[a-vx-z8][^a-vx-z8]*$/)
      return true if last_heading_subfield_index && field.subfields[last_heading_subfield_index].value =~ punct_regex
    end
    false
  end

  def lowercase_headings?(record)
    record.fields.each do |field|
      next unless field.tag =~ /[1678]../

      return true if field['a'] =~ /^[a-z]{3,}/
    end
    false
  end

  def subf_0_uri?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag =~ /^[^9]/ && field['0']

      field.subfields.each do |subfield|
        return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/
      end
    end
    false
  end

  def bad_uri?(record)
    target_fields = record.fields('856')
    return false if target_fields.empty?

    target_fields.each do |field|
      next unless field['u']

      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        string = subfield.value
        return true unless URI.escape(URI.unescape(string).scrub) == string
      end
    end
    false
  end

  ### Normalize to the NFC (combined) form of diacritics for characters with
  #     Arabic diacritics; normalize to NFD for characters below U+622 and
  #     between U+1E00 and U+2A28
  def composed_chars_normalize(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      field_index = record.fields.index(field)
      curr_subfield = 0
      field.subfields.each do |subfield|
        prevalue = subfield.value
        if prevalue =~ /^.*[\u0653\u0654\u0655].*$/
          prevalue = prevalue.unicode_normalize(:nfc)
        end
        fixed_subfield = prevalue.codepoints.map do |codepoint|
          char = codepoint.chr(Encoding::UTF_8)
          char.unicode_normalize!(:nfd) if codepoint < 1570 || (7_680..10_792).cover?(codepoint)
          char
        end.join
        record.fields[field_index].subfields[curr_subfield].value = fixed_subfield
        curr_subfield += 1
      end
    end
    record
  end

  ### Replace empty indicators with a space;
  ###   scrub indicators with bad UTF-8;
  ###   The ruby-marc gem converts nil subfields to spaces
  def empty_indicator_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      ind1_value = field.indicator1.dup
      ind1_value.scrub!('')
      field.indicator1 = ' ' if ind1_value.empty?
      ind2_value = field.indicator2.dup
      ind2_value.scrub!('')
      field.indicator2 = ' ' if ind2_value.empty?
    end
    record
  end

  ### Remove empty subfields from DataFields
  def empty_subfield_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      field.subfields.delete_if { |subfield| subfield.value.nil? || subfield.value.empty? }
    end
    record.fields.delete_if { |field| field.instance_of?(MARC::DataField) && field.subfields.empty? }
    record
  end

  ### Remove the (uri) prefix from subfield 0s
  def subf_0_uri_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag[0] != '9' && field['0']

      field.subfields.each do |subfield|
        next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/

        subfield.value = subfield.value.dup.delete_prefix('(uri)')
      end
    end
    record
  end

  ### Escape URIs
  def uri_escape(record)
    target_fields = record.fields('856')
    return record if target_fields.empty?

    fixed_record = record
    target_fields.each do |field|
      next unless field['u']

      field_index = fixed_record.fields.index(field)
      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        subfield_index = field.subfields.index(subfield)
        string = subfield.value
        fixed_string = URI.escape(URI.unescape(string).scrub)
        fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string
      end
    end
    fixed_record
  end

  ### Make the 040 $b 'eng' if it doesn't have a value
  def fix_040b(record)
    return record unless record.fields('040').size == 1

    f040 = record['040']
    field_index = record.fields.index(f040)
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return record unless b040.empty?

    subf_codes = f040.subfields.map(&:code)
    subf_index = if f040['a']
                   (subf_codes.index { |i| i == 'a' }) + 1
                 else
                   0
                 end
    subf_b = MARC::Subfield.new('b', 'eng')
    record.fields[field_index].subfields.insert(subf_index, subf_b)
    record
  end

  ### Split up subfields that contain multiple 3-letter language codes
  def fix_f041(record)
    f041 = record.fields('041')
    return record if f041.empty?

    f041.each do |field|
      f_index = record.fields.index(field)
      new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)
      field.subfields.each do |subfield|
        code = subfield.code
        val = subfield.value
        if (val.size % 3).zero?
          langs = val.scan(/.../)
          langs.each do |lang|
            new_field.append(MARC::Subfield.new(code, lang))
          end
        else
          new_field.append(MARC::Subfield.new(code, val))
        end
      end
      record.fields[f_index] = new_field
    end
    record
  end

  ### Removes text from the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def remove_prefix_from_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.delete_prefix(string)
        end
      end
    end
    record
  end

  ### Adds text to the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def add_prefix_to_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.prepend(string)
        end
      end
    end
    record
  end

  ### Sort subfields for target fields with an arbitrary order
  def subfield_sort(record, target_tags, order_array = nil)
    target_fields = record.fields.select { |f| target_tags.include?(f.tag) }
    return record if target_fields.empty?

    target_fields.each do |field|
      next unless field.class == MARC::DataField

      orig_codes = field.subfields.map { |subfield| subfield.code }.uniq.sort
      order_array = orig_codes if order_array.nil?
      new_subfields = []
      order_array.each do |code|
        next unless orig_codes.include?(code)

        target_subf = field.subfields.select { |subfield| subfield.code == code }
        target_subf.each { |subfield| new_subfields << subfield }
      end
      rem_subfields = field.subfields.select { |subf| !order_array.include?(subf.code) }
      rem_subfields.each do |subfield|
        new_subfields << subfield
      end
      field.subfields = new_subfields
    end
    record
  end
end

1	# frozen_string_literal: true
2
3	module MarcCleanup	1✔
4	### Remove non-numerical strings and append a new 020$q with the string
5	def new_020_q(record)	1✔
6	record.fields('020').each do \|f020\|	1✔
7	f020.subfields.each do \|subfield\|	1✔
8	next unless subfield.code == 'a'	2✔
9	isbn_parts = /^\s([\d\-]+)\s(\(.?\))\s$/.match(subfield.value)	1✔
10	next if isbn_parts.nil?	1✔
11	subfield.value = isbn_parts[1]	1✔
12	f020.append(MARC::Subfield.new('q', isbn_parts[2]))	1✔
13	end
14	end
15	record	1✔
16	end
17
18	### Convert ISBN-10 to ISBN-13
19	def isbn10_to_13(isbn)	1✔
20	stem = isbn[0..8]	2✔
21	return nil if stem =~ /\D/	2✔
22
23	existing_check = isbn[9]	2✔
24	return nil if existing_check && existing_check != checkdigit_10(stem)	2✔
25
26	main = ISBN13PREFIX + stem	2✔
27	checkdigit = checkdigit_13(main)	2✔
28	main + checkdigit	2✔
29	end
30
31	### Calculate check digit for ISBN-10
32	def checkdigit_10(stem)	1✔
33	int_index = 0	1✔
34	int_sum = 0	1✔
35	stem.each_char do \|digit\|	1✔
36	int_sum += digit.to_i * (10 - int_index)	9✔
37	int_index += 1	9✔
38	end
39	mod = (11 - (int_sum % 11)) % 11	1✔
40	mod == 10 ? 'X' : mod.to_s	1✔
41	end
42
43	### Calculate check digit for ISBN-13
44	def checkdigit_13(stem)	1✔
45	int_index = 0	4✔
46	int_sum = 0	4✔
47	stem.each_char do \|digit\|	4✔
48	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	48✔
49	int_index += 1	48✔
50	end
51	((10 - (int_sum % 10)) % 10).to_s	4✔
52	end
53
54	### Normalize ISBN-13
55	def isbn13_normalize(raw_isbn)	1✔
56	int_sum = 0	2✔
57	stem = raw_isbn[0..11]	2✔
58	return nil if stem =~ /\D/	2✔
59
60	int_index = 0	2✔
61	stem.each_char do \|digit\|	2✔
62	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	24✔
63	int_index += 1	24✔
64	end
65	checkdigit = checkdigit_13(stem)	2✔
66	return nil if raw_isbn[12] && raw_isbn[12] != checkdigit	2✔
67
68	stem + checkdigit	1✔
69	end
70
71	### Normalize any given string that is supposed to include an ISBN
72	def isbn_normalize(isbn)	1✔
73	return nil unless isbn	4✔
74
75	raw_isbn = isbn.dup	4✔
76	raw_isbn.delete!('-')	4✔
77	raw_isbn.delete!('\\')	4✔
78	raw_isbn.gsub!(/\([^\)]*\)/, '')	4✔
79	raw_isbn.gsub!(/^(.)\$c.$/, '\1')	4✔
80	raw_isbn.gsub!(/^(.)\$q.$/, '\1')	4✔
81	raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')	4✔
82	if raw_isbn =~ /^978/	4✔
83	raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')	2✔
84	raw_isbn.delete!(' ')	2✔
85	else
86	raw_isbn.gsub!(/([0-9])\s([0-9]{4})\s([0-9]{4})\s([0-9xX]).$/, '\1\2\3\4')	2✔
87	end
88	raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')	4✔
89	raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')	4✔
90	if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/	4✔
91	raw_isbn = raw_isbn.ljust(9, '0')	1✔
92	end
93	valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits	4✔
94	return nil unless valid_lengths.include? raw_isbn.length	4✔
95
96	if raw_isbn.length < 12	4✔
97	isbn10_to_13(raw_isbn)	2✔
98	else
99	isbn13_normalize(raw_isbn)	2✔
100	end
101	end
102
103	### If the ISBN is invalid, change the subfield code to z
104	### Otherwise, replace ISBN with normalized ISBN
105	def move_invalid_isbn(record)	1✔
106	record.fields('020').each do \|f020\|	4✔
107	f020.subfields.each do \|subfield\|	4✔
108	next unless subfield.code == 'a'	4✔
109	isbn = subfield.value	4✔
110	normalized_isbn = isbn_normalize(isbn)	4✔
111	if normalized_isbn	4✔
112	subfield.value = normalized_isbn	3✔
113	else
114	subfield.code = 'z'	1✔
115	end
116	end
117	end
118	record	4✔
119	end
120
121	# check the 041 field for errors
122	# 041 is a language code
123	def f041_errors?(record)	1✔
124	f041 = record.fields('041')	1✔
125	return false if f041.empty?	1✔
126
127	f041.each do \|field\|	1✔
128	field.subfields.each do \|subfield\|	1✔
129	val = subfield.value	1✔
130	return true if (val.size > 3) && (val.size % 3).zero?	1✔
131	end
132	end
133	false	1✔
134	end
135
136	# http://www.loc.gov/standards/valuelist/marcauthen.html
137	def auth_codes_f042	1✔
138	%w[	3✔
139	anuc croatica dc dhca dlr
140	gamma gils gnd1 gnd2 gnd3 gnd4 gnd5 gnd6 gnd7 gndz isds/c issnuk
141	lacderived lc lcac lccopycat lccopycat-nm lcd lcderive
142	lchlas lcllh lcnccp lcnitrate lcnuc lcode
143	msc natgaz nbr nlc nlmcopyc norbibl nsdp nst ntccf nznb
144	pcc premarc reveal sanb scipio toknb
145	ukblcatcopy ukblderived ukblproject ukblsr ukscp
146	xisds/c xissnuk xlc xnlc xnsdp
147	]
148	end
149
150	def auth_code_error?(record)	1✔
151	return false unless record['042']	4✔
152	return true if record.fields('042').size > 1	3✔
153
154	record['042'].subfields.each do \|subfield\|	2✔
155	next if subfield.code != 'a'	3✔
156	return true unless auth_codes_f042.include?(subfield.value)	3✔
157	end
158	false	1✔
159	end
160
161	def empty_subfields?(record)	1✔
162	record.fields.each do \|field\|	2✔
163	next unless field.instance_of?(MARC::DataField)	2✔
164
165	field.subfields.each do \|subfield\|	2✔
166	return true if subfield.value =~ /^[[:blank:]]*$/	3✔
167	end
168	end
169	false	1✔
170	end
171
172	def extra_spaces?(record)	1✔
173	blank_regex = /^.[[:blank:]]{2,}.$\|^.[[:blank:]]+$\|^[[:blank:]]+(.)$/	4✔
174	record.fields.each do \|field\|	4✔
175	next unless field.instance_of?(MARC::DataField) && field.tag != '010'	4✔
176
177	case field.tag	4✔
178	when /[1-469]..\|0[2-9].\|01[1-9]\|7[0-5].\|5[0-24-9].\|53[0-24-9]/
179	field.subfields.each do \|subfield\|	1✔
180	return true if subfield.value =~ blank_regex	1✔
181	end
182	when '533'
183	field.subfields.each do \|subfield\|	1✔
184	next if subfield.code == '7'	2✔
185
186	return true if subfield.value =~ blank_regex	1✔
187	end
188	when /7[6-8]./
189	field.subfields.each do \|subfield\|	1✔
190	next unless subfield.code =~ /[a-v3-8]/	1✔
191
192	return true if subfield.value =~ blank_regex	1✔
193	end
194	when /8../
195	field.subfields.each do \|subfield\|	1✔
196	next unless subfield.code =~ /[^w7]/	1✔
197
198	return true if subfield.value =~ blank_regex	1✔
199	end
200	end
201	end
202	false	2✔
203	end
204
205	def extra_space_gsub(string)	1✔
206	string.gsub!(/([[:blank:]]){2,}/, '\1')	4✔
207	string.gsub!(/^(.*)[[:blank:]]+$/, '\1')	4✔
208	string.gsub(/^[[:blank:]]+(.*)$/, '\1')	4✔
209	end
210
211	### Remove extra spaces from all fields that are not positionally defined
212	def extra_space_fix(record)	1✔
213	record.fields.each do \|field\|	4✔
214	next unless field.instance_of?(MARC::DataField) && field.tag != '010'	4✔
215
216	case field.tag	4✔
217	when /^[1-469]..\|0[2-9].\|01[1-9]\|7[0-5].\|5[0-24-9].\|53[0-24-9]/
218	field.subfields.each do \|subfield\|	1✔
219	next if subfield.value.nil?	1✔
220
221	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
222	end
223	when '533'
224	field.subfields.each do \|subfield\|	1✔
225	next if subfield.code == '7' \|\| subfield.value.nil?	2✔
226
227	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
228	end
229	when /^7[6-8]./
230	field.subfields.each do \|subfield\|	1✔
231	next if subfield.code =~ /[^a-v3-8]/ \|\| subfield.value.nil?	1✔
232
233	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
234	end
235	when /^8../
236	field.subfields.each do \|subfield\|	1✔
237	next if %w[w 7].include?(subfield.code) \|\| subfield.value.nil?	1✔
238
239	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
240	end
241	end
242	end
243	record	4✔
244	end
245
246	def multiple_no_040?(record)	1✔
247	record.fields('040').size != 1	1✔
248	end
249
250	def multiple_no_040b?(record)	1✔
251	f040 = record.fields('040')	3✔
252	return true if f040.size != 1	3✔
253
254	f040 = f040.first	3✔
255	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	7✔
256	return true if b040.size != 1	3✔
257
258	b040.first.value.match?(/^\s*$/)	2✔
259	end
260
261	def f046_errors?(record)	1✔
262	subf_codes = %w[b c d e]	4✔
263	subf_a_values = %w[r s p t x q n i k r m t x n]	4✔
264	f046 = record.fields('046')	4✔
265	return false if f046.empty?	4✔
266
267	f046.each do \|field\|	3✔
268	codes = field.subfields.map(&:code)	3✔
269	return true if field['a'] && !subf_a_values.include?(field['a'])	3✔
270	return true if field['a'].nil? && (subf_codes & codes).size.positive?	2✔
271	end
272	false	1✔
273	end
274
275	def multiple_no_245?(record)	1✔
276	record.fields('245').size != 1	24✔
277	end
278
279	def missing_040c?(record)	1✔
280	return true unless record['040'] && record['040']['c']	2✔
281
282	false	1✔
283	end
284
285	def pair_880_errors?(record)	1✔
286	return true if record.fields('880').select { \|field\| field['6'].nil? }.size.positive?	12✔
287
288	pair_880s = f880_pairings(record)	5✔
289	linked_fields = linked_field_pairings(record)	5✔
290	pair_880s.uniq != pair_880s \|\| pair_880s.uniq.sort != linked_fields.uniq.sort	5✔
291	end
292
293	def f880_pairings(record)	1✔
294	target_fields = record.fields('880').select do \|field\|	5✔
295	field['6'] =~ /^[0-9]{3}-[0-9][1-9]/	5✔
296	end
297	target_fields.map do \|field\|	5✔
298	field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')	4✔
299	end
300	end
301
302	def linked_field_pairings(record)	1✔
303	target_fields = record.fields('010'..'899').select do \|field\|	5✔
304	field.tag != '880' && field['6']	10✔
305	end
306	target_fields.map do \|field\|	5✔
307	"#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1')}"	3✔
308	end
309	end
310
311	def has_130_240?(record)	1✔
312	(%w[130 240] - record.tags).empty?	24✔
313	end
314
315	def multiple_1xx?(record)	1✔
316	record.fields('100'..'199').size > 1	24✔
317	end
318
319	def relator_chars?(record)	1✔
320	record.fields.each do \|field\|	×
321	if field.tag =~ /[17][01]0/	×
322	field.subfields.each do \|subfield\|	×
323	next unless subfield.code == 'e' && subfield.value =~ /.[^a-z, \.]./	×
324
325	return true	×
326	end
327	elsif field.tag =~ /[17]11/	×
328	field.subfields.each do \|subfield\|	×
329	next unless subfield.code == 'j' && subfield.value =~ /.[^a-z, \.]./	×
330
331	return true	×
332	end
333	end
334	end
335	false	×
336	end
337
338	def x00_subfq?(record)	1✔
339	record.fields(%w[100 600 700 800]).each do \|field\|	×
340	field.subfields.each do \|subfield\|	×
341	next unless subfield.code == 'q' && subfield.value =~ /^(?!\([^\)]*\))$/	×
342
343	return true	×
344	end
345	end
346	false	×
347	end
348
349	def no_comma_x00?(record)	1✔
350	record.fields(%w[100 600 700 800]).each do \|field\|	×
351	code_array = ''	×
352	field.subfields.each do \|subfield\|	×
353	code_array << subfield.code	×
354	end
355	subfx_d_index = code_array.index(/.d/)	×
356	return true if subfx_d_index && field.subfields[subfx_d_index].value =~ /^.*[^,]$/	×
357	end
358	false	×
359	end
360
361	def relator_comma?(record)	1✔
362	comma_regex = /^.*[^,]$/	×
363	record.fields.each do \|field\|	×
364	next unless field.tag =~ /[17][01][01]/	×
365
366	code_array = ''	×
367	field.subfields.each do \|subfield\|	×
368	code_array << subfield.code	×
369	end
370	if field.tag =~ /[17][01]0/	×
371	subfx_e_index = code_array.index(/.e/)	×
372	return true if subfx_e_index && field.subfields[subfx_e_index].value =~ comma_regex	×
373	elsif field.tag =~ /[17]11/	×
374	subfx_j_index = code_array.index(/.j/)	×
375	return true if subfx_j_index && field.subfields[subfx_j_index].value =~ comma_regex	×
376	end
377	end
378	false	×
379	end
380
381	def heading_end_punct?(record)	1✔
382	punct_regex = /.*[^"\).\!\?\-]$/	×
383	record.fields.each do \|field\|	×
384	next unless field.tag =~ /^[167][0-5].$/ && field.indicator2 =~ /[^47]/	×
385
386	code_array = ''	×
387	field.subfields.each do \|subfield\|	×
388	code_array << subfield.code	×
389	end
390	last_heading_subfield_index = code_array.index(/[a-vx-z8][^a-vx-z8]*$/)	×
391	return true if last_heading_subfield_index && field.subfields[last_heading_subfield_index].value =~ punct_regex	×
392	end
393	false	×
394	end
395
396	def lowercase_headings?(record)	1✔
397	record.fields.each do \|field\|	×
398	next unless field.tag =~ /[1678]../	×
399
400	return true if field['a'] =~ /^[a-z]{3,}/	×
401	end
402	false	×
403	end
404
405	def subf_0_uri?(record)	1✔
406	record.fields.each do \|field\|	3✔
407	next unless field.instance_of?(MARC::DataField) && field.tag =~ /^[^9]/ && field['0']	3✔
408
409	field.subfields.each do \|subfield\|	2✔
410	return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/	4✔
411	end
412	end
413	false	2✔
414	end
415
416	def bad_uri?(record)	1✔
417	target_fields = record.fields('856')	×
418	return false if target_fields.empty?	×
419
420	target_fields.each do \|field\|	×
421	next unless field['u']	×
422
423	field.subfields.each do \|subfield\|	×
424	next unless subfield.code == 'u'	×
425
426	string = subfield.value	×
427	return true unless URI.escape(URI.unescape(string).scrub) == string	×
428	end
429	end
430	false	×
431	end
432
433	### Normalize to the NFC (combined) form of diacritics for characters with
434	# Arabic diacritics; normalize to NFD for characters below U+622 and
435	# between U+1E00 and U+2A28
436	def composed_chars_normalize(record)	1✔
437	record.fields.each do \|field\|	2✔
438	next unless field.class == MARC::DataField	2✔
439
440	field_index = record.fields.index(field)	2✔
441	curr_subfield = 0	2✔
442	field.subfields.each do \|subfield\|	2✔
443	prevalue = subfield.value	4✔
444	if prevalue =~ /^.[\u0653\u0654\u0655].$/	4✔
445	prevalue = prevalue.unicode_normalize(:nfc)	1✔
446	end
447	fixed_subfield = prevalue.codepoints.map do \|codepoint\|	4✔
448	char = codepoint.chr(Encoding::UTF_8)	84✔
449	char.unicode_normalize!(:nfd) if codepoint < 1570 \|\| (7_680..10_792).cover?(codepoint)	84✔
450	char	84✔
451	end.join
452	record.fields[field_index].subfields[curr_subfield].value = fixed_subfield	4✔
453	curr_subfield += 1	4✔
454	end
455	end
456	record	2✔
457	end
458
459	### Replace empty indicators with a space;
460	### scrub indicators with bad UTF-8;
461	### The ruby-marc gem converts nil subfields to spaces
462	def empty_indicator_fix(record)	1✔
463	record.fields.each do \|field\|	2✔
464	next unless field.instance_of?(MARC::DataField)	2✔
465
466	ind1_value = field.indicator1.dup	2✔
467	ind1_value.scrub!('')	2✔
468	field.indicator1 = ' ' if ind1_value.empty?	2✔
469	ind2_value = field.indicator2.dup	2✔
470	ind2_value.scrub!('')	2✔
471	field.indicator2 = ' ' if ind2_value.empty?	2✔
472	end
473	record	2✔
474	end
475
476	### Remove empty subfields from DataFields
477	def empty_subfield_fix(record)	1✔
478	record.fields.each do \|field\|	1✔
479	next unless field.instance_of?(MARC::DataField)	2✔
480
481	field.subfields.delete_if { \|subfield\| subfield.value.nil? \|\| subfield.value.empty? }	11✔
482	end
483	record.fields.delete_if { \|field\| field.instance_of?(MARC::DataField) && field.subfields.empty? }	3✔
484	record	1✔
485	end
486
487	### Remove the (uri) prefix from subfield 0s
488	def subf_0_uri_fix(record)	1✔
489	record.fields.each do \|field\|	3✔
490	next unless field.instance_of?(MARC::DataField) && field.tag[0] != '9' && field['0']	3✔
491
492	field.subfields.each do \|subfield\|	2✔
493	next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/	4✔
494
495	subfield.value = subfield.value.dup.delete_prefix('(uri)')	1✔
496	end
497	end
498	record	3✔
499	end
500
501	### Escape URIs
502	def uri_escape(record)	1✔
503	target_fields = record.fields('856')	×
504	return record if target_fields.empty?	×
505
506	fixed_record = record	×
507	target_fields.each do \|field\|	×
508	next unless field['u']	×
509
510	field_index = fixed_record.fields.index(field)	×
511	field.subfields.each do \|subfield\|	×
512	next unless subfield.code == 'u'	×
513
514	subfield_index = field.subfields.index(subfield)	×
515	string = subfield.value	×
516	fixed_string = URI.escape(URI.unescape(string).scrub)	×
517	fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string	×
518	end
519	end
520	fixed_record	×
521	end
522
523	### Make the 040 $b 'eng' if it doesn't have a value
524	def fix_040b(record)	1✔
525	return record unless record.fields('040').size == 1	2✔
526
527	f040 = record['040']	2✔
528	field_index = record.fields.index(f040)	2✔
529	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	4✔
530	return record unless b040.empty?	2✔
531
532	subf_codes = f040.subfields.map(&:code)	2✔
533	subf_index = if f040['a']	2✔
534	(subf_codes.index { \|i\| i == 'a' }) + 1	2✔
535	else
536	0	1✔
537	end
538	subf_b = MARC::Subfield.new('b', 'eng')	2✔
539	record.fields[field_index].subfields.insert(subf_index, subf_b)	2✔
540	record	2✔
541	end
542
543	### Split up subfields that contain multiple 3-letter language codes
544	def fix_f041(record)	1✔
545	f041 = record.fields('041')	2✔
546	return record if f041.empty?	2✔
547
548	f041.each do \|field\|	2✔
549	f_index = record.fields.index(field)	2✔
550	new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)	2✔
551	field.subfields.each do \|subfield\|	2✔
552	code = subfield.code	2✔
553	val = subfield.value	2✔
554	if (val.size % 3).zero?	2✔
555	langs = val.scan(/.../)	1✔
556	langs.each do \|lang\|	1✔
557	new_field.append(MARC::Subfield.new(code, lang))	3✔
558	end
559	else
560	new_field.append(MARC::Subfield.new(code, val))	1✔
561	end
562	end
563	record.fields[f_index] = new_field	2✔
564	end
565	record	2✔
566	end
567
568	### Removes text from the beginning of a subfield
569	### An array of hashes of the format { field:, subfields: } will be passed
570	### in the targets: symbol
571	### subfield: is an array of subfield codes
572	def remove_prefix_from_subfield(record:, targets:, string:)	1✔
573	targets.each do \|target\|	1✔
574	record.fields(target[:field]).each do \|field\|	1✔
575	field.subfields.each do \|subfield\|	1✔
576	next unless target[:subfields].include?(subfield.code)	2✔
577
578	subfield.value = subfield.value.dup.delete_prefix(string)	1✔
579	end
580	end
581	end
582	record	1✔
583	end
584
585	### Adds text to the beginning of a subfield
586	### An array of hashes of the format { field:, subfields: } will be passed
587	### in the targets: symbol
588	### subfield: is an array of subfield codes
589	def add_prefix_to_subfield(record:, targets:, string:)	1✔
590	targets.each do \|target\|	1✔
591	record.fields(target[:field]).each do \|field\|	1✔
592	field.subfields.each do \|subfield\|	1✔
593	next unless target[:subfields].include?(subfield.code)	2✔
594
595	subfield.value = subfield.value.dup.prepend(string)	1✔
596	end
597	end
598	end
599	record	1✔
600	end
601
602	### Sort subfields for target fields with an arbitrary order
603	def subfield_sort(record, target_tags, order_array = nil)	1✔
604	target_fields = record.fields.select { \|f\| target_tags.include?(f.tag) }	×
605	return record if target_fields.empty?	×
606
607	target_fields.each do \|field\|	×
608	next unless field.class == MARC::DataField	×
609
610	orig_codes = field.subfields.map { \|subfield\| subfield.code }.uniq.sort	×
611	order_array = orig_codes if order_array.nil?	×
612	new_subfields = []	×
613	order_array.each do \|code\|	×
614	next unless orig_codes.include?(code)	×
615
616	target_subf = field.subfields.select { \|subfield\| subfield.code == code }	×
617	target_subf.each { \|subfield\| new_subfields << subfield }	×
618	end
619	rem_subfields = field.subfields.select { \|subf\| !order_array.include?(subf.code) }	×
620	rem_subfields.each do \|subfield\|	×
621	new_subfields << subfield	×
622	end
623	field.subfields = new_subfields	×
624	end
625	record	×
626	end
627	end

pulibrary / marc_cleanup / 6993595b-7e17-4f96-a203-b3db1eaf8e8a

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous