process_hash_row - helper VALIDATE HASHES 
 Converts hash keys and vals into parsed line.
355910
def process_hash_row(hsh)
      if @headers.any?
        keys_or_values = hsh.values
        @row_id = hsh[:row_id]
      else
        keys_or_values = hsh.keys.map(&:to_s)
      end

      file_line = keys_or_values.join(',')
      validated_line = utf_filter(check_utf(file_line))
      res = line_parse(validated_line)
      res
    end
511107
def _validate_entries(self, processes):
        
        errors = list()

        # First we'll make sure there's no mismatch between the filesystem
        # and the list of files in the manifest(s)
        only_in_manifests, only_on_fs = self.compare_manifests_with_fs()
        for path in only_in_manifests:
            e = FileMissing(path)
            logger.warning(str(e))
            errors.append(e)
        for path in only_on_fs:
            e = UnexpectedFile(path)
            logger.warning(str(e))
            errors.append(e)

        # To avoid the overhead of reading the file more than once or loading
        # potentially massive files into memory we'll create a dictionary of
        # hash objects so we can open a file, read a block and pass it to
        # multiple hash objects

        available_hashers = set()
        for alg in self.algs:
            try:
                hashlib.new(alg)
                available_hashers.add(alg)
            except ValueError:
                logger.warning("Unable to validate file contents using unknown %s hash algorithm", alg)

        if not available_hashers:
            raise RuntimeError("%s: Unable to validate bag contents: none of the hash algorithms in %s are supported!" % (self, self.algs))

        def _init_worker():
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        args = ((self.path, rel_path, hashes, available_hashers) for rel_path, hashes in list(self.entries.items()))

        try:
            if processes == 1:
                hash_results = list(map(_calc_hashes, args))
            else:
                try:
                    pool = multiprocessing.Pool(processes if processes else None, _init_worker)
                    hash_results = pool.map(_calc_hashes, args)
                finally:
                    try:
                        pool.terminate()
                    except:
                        # we really don't care about any exception in terminate()
                        pass
        # Any unhandled exceptions are probably fatal
        except:
            logger.exception("unable to calculate file hashes for %s", self)
            raise

        for rel_path, f_hashes, hashes in hash_results:
            for alg, computed_hash in list(f_hashes.items()):
                stored_hash = hashes[alg]
                if stored_hash.lower() != computed_hash:
                    e = ChecksumMismatch(rel_path, alg, stored_hash.lower(), computed_hash)
                    logger.warning(str(e))
                    errors.append(e)

        if errors:
            raise BagValidationError("invalid bag", errors)
1140212
def process_hash(tags, property, content, **opts)
      content.each do |key, value|
        key = key.to_s == '_' ? property : "#{property}:#{key}"
        value = normalized_meta_tags[value] if value.kind_of?(Symbol)
        process_tree(tags, key, value, **opts)
      end
    end
367605
def _sanitize_value(key, value)
      case value
      when String
        value.dup.freeze
      when Symbol, Integer, Float, NilClass, TrueClass, FalseClass
        value
      when Array
        _sanitize_array(key, value)
      when Hash
        _sanitize_params(value)
      else
        raise Errors::UnsafeValueError.new(key, value)
      end
    end
1320796
def parse_hash(hash, convert_types = false)
      new_hash = {}
      hash.each do |k, v|
        if convert_types
          v = v.to_i if v && v.match(/^\d+$/)
          v = true if v == 'true'
          v = false if v == 'false'
          v = nil if v == 'nil'
        end
        if k =~ /\./
          sub_hash = new_hash
          split = k.split('.')
          split.each do |sub_key|
            if sub_key == split.last
              sub_hash[sub_key] = v
            else
              sub_hash[sub_key] ||= {}
              sub_hash = sub_hash[sub_key]
            end
          end
          new_hash[split.first] ||= {}
        else
          new_hash[k] = v
        end
      end
      new_hash
    end
1168899
def to_params_hash(keys,values)
      hash = {}
      keys.size.times { |i| hash[ keys[i].to_sym ] = values[i] }
      hash
    end
419680
def extract_key_value(line)
      match = line.match(KEY_VALUE_PATTERN)
      if match
        key = match[1]
        value = match[2]
        [key.strip, value.strip]
      else
        []
      end
    end
155857
def validate_csv(file_path)
      return unless file_path.present?
      File.open(file_path).each do |file_line|
        validated_line = utf_filter(check_utf(file_line))
        @row_id += 1
        if validated_line
          CSV.parse(validated_line) do |row|
            if @headers.empty?
              @headers = row
            else
              @data_hash.merge!(row_to_hsh(row))
              @valid_rows << @data_hash
            end
          end
        end
      rescue StandardError => error
        @error_rows << { row_id: @row_id, text: error.message }
      end
      utf_results = compile_results
    end
187731
def hash_to_rets_query_string(hash)
      [].tap do |str|
        hash.each_pair do |k,v|
          val = value_map(v)
          str << "(#{k.to_s.camelize}=#{val})"
        end
      end.join(',')
    end
950170
def validate_headers(row)
      valid_headers = @config.fields - @config.export_only_fields
      #valid_headers += @config.has_ones.map

      row.headers.compact.map(&:to_sym).each do |h|
        unless valid_headers.include?(h)
          logger.warn "ignoring unknown column #{h}"
        end
      end
    end
1316329
def _HashRow(cls, row):
    
    values = []
    for value in row:
      try:
        value = '{0!s}'.format(value)
      except UnicodeDecodeError:
        # In Python 2, blobs are "read-write buffer" and will cause a
        # UnicodeDecodeError exception if we try format it as a string.
        # Since Python 3 does not support the buffer type we cannot check
        # the type of value.
        value = repr(value)

      values.append(value)

    return hash(' '.join(values))
314020
def normalize!(params)
      case params
      when Hash
        params.keys.each do |k|
          params[k.to_s] = params.delete(k)
          normalize!(params[k.to_s])
        end
      when Array
        params.map! do |el|
          normalize!(el)
        end
      end
      params
    end
848046
def to_h(with_errors: false, with_value: true)
      hash = {
        name: name,
        label: label,
        type: type,
        description: description,
        default: default,
        from: from.reject { |k,_| k == :default},
        to: to
      }.merge(type_options).reject { |_,v| v.nil? }
      hash[:skip_if] = skip_if if skip_if
      hash[:only_if] = only_if if only_if
      hash[:errors]  = errors  if with_errors
      if with_value
        if type == 'group'
          hash[:value]   = value.to_h(with_values: true, with_errors: with_errors)
        else
          hash[:value] = value
        end
      end
      hash
    end
220922
def trim_and_retry(buf)
      *lines, last = buf

      # For inner Array or Hash or Arguments list.
      lines << last.gsub(/,\s*$/, "")
      do_parse("a(#{lines.join})") # wrap dummy method
    rescue ::Parser::SyntaxError
    end
613997
def hashes
      all_rows = rows.locate
      header_row = all_rows.first || raise(Error, 'no rows in table')

      all_rows.entries[1..-1].map do |row|
        cell_size_check(header_row, row)
        Hash[headers(header_row).map(&:text).zip(row.cells.map(&:text))]
      end
    end
1291185
def is_valid_row(cls, row):
        """"""
        for k in row.keys():
            if row[k] is None:
                return False
        return True
256625
def postprocess(self, json_string):
        """"""
        is_compressing, is_hash, compressed, spaces = False, False, [], 0
        for row in json_string.split('\n'):
            if is_compressing:
                if (row[:spaces + 5] == ' ' * (spaces + 4) +
                        ('"' if is_hash else '{')):
                    compressed.append(row.rstrip())
                elif (len(row) > spaces and row[:spaces] == ' ' * spaces and
                        re.match('[\]\}],?', row[spaces:].rstrip())):
                    compressed.append(row.rstrip())
                    is_compressing = False
                else:
                    compressed[-1] += ' ' + row.strip()
            else:
                compressed.append(row.rstrip())
                if any(a in row for a in ['edges', 'nodes']):
                    # Fix to handle issues that arise with empty lists
                    if '[]' in row:
                        continue
                    spaces = sum(1 for _ in takewhile(str.isspace, row))
                    is_compressing, is_hash = True, '{' in row
        return '\n'.join(compressed)